@@ -23,10 +23,14 @@ along with GCC; see the file COPYING3. If not see
extern void asan_function_start (void);
extern void asan_finish_file (void);
+extern void hwasan_finish_file (void);
extern void hwasan_record_base (rtx);
+extern uint8_t hwasan_current_tag ();
extern void hwasan_increment_tag ();
extern rtx hwasan_with_tag (rtx, poly_int64);
extern void hwasan_tag_init ();
+extern rtx hwasan_create_untagged_base (rtx);
+extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
extern bool memory_tagging_p (void);
extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
HOST_WIDE_INT *, tree *, int);
@@ -2895,6 +2895,11 @@ initialize_sanitizer_builtins (void)
= build_function_type_list (void_type_node, uint64_type_node,
ptr_type_node, NULL_TREE);
+ tree BT_FN_VOID_PTR_UINT8_SIZE
+ = build_function_type_list (void_type_node, ptr_type_node,
+ unsigned_char_type_node, size_type_node,
+ NULL_TREE);
+
tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
tree BT_FN_IX_CONST_VPTR_INT[5];
tree BT_FN_IX_VPTR_IX_INT[5];
@@ -2945,6 +2950,8 @@ initialize_sanitizer_builtins (void)
#define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
#define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
#define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
+#undef ATTR_NOTHROW_LIST
+#define ATTR_NOTHROW_LIST ECF_NOTHROW
#undef ATTR_NOTHROW_LEAF_LIST
#define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
#undef ATTR_TMPURE_NOTHROW_LEAF_LIST
@@ -3707,6 +3714,8 @@ hwasan_record_base (rtx base)
hwasan_base_ptr = base;
}
+uint8_t hwasan_current_tag () { return tag_offset; }
+
void
hwasan_increment_tag ()
{
@@ -3760,4 +3769,104 @@ hwasan_tag_init ()
tag_offset = HWASAN_STACK_BACKGROUND + 1;
}
+void
+hwasan_emit_prologue (rtx *bases,
+ rtx *untagged_bases,
+ poly_int64 *offsets,
+ uint8_t *tags,
+ size_t length)
+{
+ /*
+ NOTE: bases contains both the tagged and untagged base.
+ This allows us to get both the original frame tag and the untagged variable
+ pointer with a minimal of extra instructions.
+
+ We fetch the untagged variable pointer from the offset to the untagged base
+ and fetch the "base" tag from the tagged base.
+
+ We need the untagged base pointer since libhwasan only accepts untagged
+ pointers in __hwasan_tag_memory. We need the tagged base pointer to obtain
+ the base tag for an offset.
+
+ We also will need the tagged base pointer for MTE, since the ADDTAG
+ instruction takes a tagged pointer.
+ */
+ for (size_t i = 0; (i * 2) + 1 < length; i++)
+ {
+ poly_int64 start = offsets[i * 2];
+ poly_int64 end = offsets[(i * 2) + 1];
+
+ poly_int64 bot, top;
+ if (known_ge (start, end))
+ {
+ top = start;
+ bot = end;
+ }
+ else
+ {
+ top = end;
+ bot = start;
+ }
+ poly_int64 size = (top - bot);
+
+ /* Can't check that all poly_int64's are aligned, but still nice
+ to check when we can. */
+ HOST_WIDE_INT tmp;
+ if (top.is_constant (&tmp))
+ gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+ if (bot.is_constant (&tmp))
+ gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+ if (size.is_constant (&tmp))
+ gcc_assert (tmp % HWASAN_TAG_GRANULE_SIZE == 0);
+
+ /* TODO Other options (i.e. inline options) */
+ /* TODO At the moment we don't generate a random base tag for each
+ frame. When that happens we will need to generate the tag by
+ adding tags[i] to the frame tag fetched from `bases[i]`. */
+ rtx ret = init_one_libfunc ("__hwasan_tag_memory");
+ emit_library_call (ret,
+ LCT_NORMAL,
+ VOIDmode,
+ plus_constant (ptr_mode, untagged_bases[i], bot),
+ ptr_mode,
+ const_int_rtx[MAX_SAVED_CONST_INT + tags[i]],
+ QImode,
+ gen_int_mode (size, ptr_mode),
+ ptr_mode);
+ }
+}
+
+rtx
+hwasan_create_untagged_base (rtx orig_base)
+{
+ rtx untagged_base = gen_reg_rtx (Pmode);
+ rtx tag_mask = gen_int_mode ((1ULL << HWASAN_SHIFT) - 1, Pmode);
+ untagged_base = expand_binop (Pmode, and_optab,
+ orig_base, tag_mask,
+ untagged_base, true, OPTAB_DIRECT);
+ gcc_assert (untagged_base);
+ return untagged_base;
+}
+
+/* Needs to be GTY(()), because cgraph_build_static_cdtor may
+ invoke ggc_collect. */
+static GTY(()) tree hwasan_ctor_statements;
+
+void
+hwasan_finish_file (void)
+{
+ /* Avoid instrumenting code in the hwasan constructors/destructors. */
+ flag_sanitize &= ~SANITIZE_HWADDRESS;
+ /* TODO Only do this if in userspace.
+ For kernel space will have to look more closely into this.
+ May want to look at `asan_finish_file` for what ASAN does in this
+ situation. */
+
+ int priority = MAX_RESERVED_INIT_PRIORITY - 1;
+ tree fn = builtin_decl_implicit (BUILT_IN_HWASAN_INIT);
+ append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
+ cgraph_build_static_cdtor ('I', hwasan_ctor_statements, priority);
+ flag_sanitize |= SANITIZE_HWADDRESS;
+}
+
#include "gt-asan.h"
@@ -625,6 +625,8 @@ DEF_FUNCTION_TYPE_3 (BT_FN_VOID_UINT32_UINT32_PTR,
DEF_FUNCTION_TYPE_3 (BT_FN_VOID_SIZE_SIZE_PTR, BT_VOID, BT_SIZE, BT_SIZE,
BT_PTR)
DEF_FUNCTION_TYPE_3 (BT_FN_UINT_UINT_PTR_PTR, BT_UINT, BT_UINT, BT_PTR, BT_PTR)
+DEF_FUNCTION_TYPE_3 (BT_FN_VOID_PTR_UINT8_SIZE, BT_VOID, BT_PTR, BT_UINT8,
+ BT_SIZE)
DEF_FUNCTION_TYPE_4 (BT_FN_SIZE_CONST_PTR_SIZE_SIZE_FILEPTR,
BT_SIZE, BT_CONST_PTR, BT_SIZE, BT_SIZE, BT_FILEPTR)
@@ -237,6 +237,7 @@ along with GCC; see the file COPYING3. If not see
DEF_BUILTIN (ENUM, "__builtin_" NAME, BUILT_IN_NORMAL, TYPE, TYPE, \
true, true, true, ATTRS, true, \
(flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_THREAD \
+ | SANITIZE_HWADDRESS \
| SANITIZE_UNDEFINED \
| SANITIZE_UNDEFINED_NONDEFAULT) \
|| flag_sanitize_coverage))
@@ -1034,9 +1034,24 @@ struct stack_vars_data
The vector is in reversed, highest offset pairs come first. */
auto_vec<HOST_WIDE_INT> asan_vec;
+ /* HWASAN records the poly_int64 since it needs to act on everything recorded
+ to the stack (as anything not properly coloured would end up causing a
+ falut of some sort).
+
+ ASAN records HOST_WIDE_INT offsets (that was enough before the
+ introduction of SVE vectors) which */
+ auto_vec<poly_int64> hwasan_vec;
+ auto_vec<rtx> hwasan_untagged_base_vec;
+ auto_vec<rtx> hwasan_base_vec;
+
/* Vector of partition representative decls in between the paddings. */
auto_vec<tree> asan_decl_vec;
+ /* Vector of tag offsets representing the colour of each stack variable.
+ Each offset determines the difference between the randomly generated
+ colour for the current frame and the colour for this stack variable. */
+ auto_vec<uint8_t> hwasan_colour_vec;
+
/* Base pseudo register for Address Sanitizer protected automatic vars. */
rtx asan_base;
@@ -1054,6 +1069,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
size_t si, i, j, n = stack_vars_num;
poly_uint64 large_size = 0, large_alloc = 0;
rtx large_base = NULL;
+ rtx large_untagged_base = NULL;
unsigned large_align = 0;
bool large_allocation_done = false;
tree decl;
@@ -1110,7 +1126,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
{
rtx base;
unsigned base_align, alignb;
- poly_int64 offset;
+ poly_int64 offset = 0;
i = stack_vars_sorted[si];
@@ -1156,7 +1172,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
ABI requirements) and these can't share a tag granule with a
tagged variable. */
gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
- alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+ offset = alloc_stack_frame_space (0, HWASAN_TAG_GRANULE_SIZE);
+ data->hwasan_vec.safe_push (offset);
+ data->hwasan_untagged_base_vec.safe_push (virtual_stack_vars_rtx);
}
/* ASAN description strings don't yet have a syntax for expressing
polynomial offsets. */
@@ -1237,6 +1255,9 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
base_align = crtl->max_used_stack_slot_alignment;
}
+
+ if (memory_tagging_p ())
+ data->hwasan_vec.safe_push (offset);
}
else
{
@@ -1262,6 +1283,18 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
gcc_assert (large_base != NULL);
large_alloc = aligned_upper_bound (large_alloc, alignb);
+ if (memory_tagging_p ())
+ {
+ /*
+ For now we just assume that an object with a large alignment
+ requirement means that the alignment requirement is greater
+ than the required alignment for tags.
+ */
+ if (!large_untagged_base)
+ large_untagged_base = hwasan_create_untagged_base (large_base);
+ data->hwasan_vec.safe_push (large_alloc);
+ data->hwasan_untagged_base_vec.safe_push (large_untagged_base);
+ }
offset = large_alloc;
large_alloc += stack_vars[i].size;
if (memory_tagging_p ())
@@ -1280,6 +1313,7 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
*/
poly_int64 align_again =
aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
+ data->hwasan_vec.safe_push (align_again);
}
base = large_base;
@@ -1297,8 +1331,15 @@ expand_stack_vars (bool (*pred) (size_t), struct stack_vars_data *data)
? data->asan_base
: virtual_stack_vars_rtx);
}
+
if (memory_tagging_p ())
- hwasan_increment_tag ();
+ {
+ /* Record the tag for this object in `data` so the prologue knows
+ what colour to put in the shadow memory during cfgexpand.c. */
+ data->hwasan_base_vec.safe_push (base);
+ data->hwasan_colour_vec.safe_push (hwasan_current_tag ());
+ hwasan_increment_tag ();
+ }
}
gcc_assert (known_eq (large_alloc, large_size));
@@ -2358,6 +2399,13 @@ expand_used_vars (void)
}
expand_stack_vars (NULL, &data);
+
+ if (memory_tagging_p ())
+ hwasan_emit_prologue (data.hwasan_base_vec.address (),
+ data.hwasan_untagged_base_vec.address (),
+ data.hwasan_vec.address (),
+ data.hwasan_colour_vec.address (),
+ data.hwasan_vec.length ());
}
if (asan_sanitize_allocas_p () && cfun->calls_alloca)
@@ -180,6 +180,12 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_COMPARE, "__sanitizer_ptr_cmp",
DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_SUBTRACT, "__sanitizer_ptr_sub",
BT_FN_VOID_PTR_PTRMODE, ATTR_NOTHROW_LEAF_LIST)
+/* Hardware Address Sanitizer. */
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_INIT, "__hwasan_init",
+ BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_TAG_MEM, "__hwasan_tag_memory",
+ BT_FN_VOID_PTR_UINT8_SIZE, ATTR_NOTHROW_LIST)
+
/* Thread Sanitizer */
DEF_SANITIZER_BUILTIN(BUILT_IN_TSAN_INIT, "__tsan_init",
BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
@@ -508,6 +508,9 @@ compile_file (void)
if (flag_sanitize & SANITIZE_THREAD)
tsan_finish_file ();
+ if (gate_hwasan ())
+ hwasan_finish_file ();
+
omp_finish_file ();
hsa_output_brig ();