[RFC,12/X,libsanitizer] Check pointer tags match address tags

Message ID VI1PR08MB54715A58DC26A5601AAD7C6AE0BA0@VI1PR08MB5471.eurprd08.prod.outlook.com
State New
Headers show
Series
  • [RFC,12/X,libsanitizer] Check pointer tags match address tags
Related show

Commit Message

Matthew Malcomson Sept. 6, 2019, 2:46 p.m.
This adds the `hwasan` pass that puts HWASAN_CHECK internal functions
around memory accesses to check that tags in the pointer being used
match the tag stored in shadow memory for the memory region being used.

These internal functions are expanded into actual checks in the sanopt
pass that happens just before expansion into RTL.

We use the same mechanism that currently inserts ASAN_CHECK internal
functions to insert the new HWASAN_CHECK functions.  Instrumenting
builtin function calls is not yet handled and will be in a later patch
in this series.

As yet we always terminate the program on a tag mismatch, and always
check tags via a library call to libhwasan.  Allowing a program to
continue from tag mismatch, and implementing inline checks are intended
for a later revision.

gcc/ChangeLog:

2019-09-06  Matthew Malcomson  <matthew.malcomson@arm.com>

	* asan.c (build_check_stmt): Generate HWASAN_CHECK for HWASAN.
	(instrument_derefs): Avoid checking global variables for HWASAN.
	(maybe_instrument_call): Temporarily avoid for HWASAN.
	(hwasan_instrument): New.
	(hwasan_expand_check_ifn): New.
	(gate_hwasan): New.
	(class pass_hwasan): New.
	(make_pass_hwasan): New.
	(class pass_hwasan_O0): New.
	(make_pass_hwasan_O0): New.
	* asan.h (hwasan_expand_check_ifn): New.
	(gate_hwasan): New.
	(asan_intercepted_p): No memory function is intercepted by
	libhwasan.
	(asan_sanitize_use_after_scope): True for memory tagging.
	* internal-fn.c (expand_HWASAN_CHECK): New.
	* internal-fn.def (HWASAN_CHECK): New.
	* passes.def (hwasan): New pass.
	* sanitizer.def (BUILT_IN_HWASAN_LOADN): New builtin.
	(BUILT_IN_HWASAN_STOREN): New builtin.
	* sanopt.c (sanopt_optimize_walker): Account for HWASAN_CHECK.
	(pass_sanopt::execute): Account for HWASAN_CHECK.
	* tree-pass.h (make_pass_hwasan, make_pass_hwasan_O0): New decl.



###############     Attachment also inlined for ease of reply    ###############
diff --git a/gcc/asan.h b/gcc/asan.h
index c5492ce35980d0b26d4707f96482b69dc76a525a..68ea1b4afaf9195553251a987df33788421fa142 100644
--- a/gcc/asan.h
+++ b/gcc/asan.h
@@ -32,7 +32,9 @@ extern void hwasan_tag_init ();
 extern rtx hwasan_create_untagged_base (rtx);
 extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
 extern rtx_insn *hwasan_emit_uncolour_frame (rtx, rtx, rtx_insn *);
+extern bool hwasan_expand_check_ifn (gimple_stmt_iterator *, bool);
 extern bool memory_tagging_p (void);
+extern bool gate_hwasan (void);
 extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
 					     HOST_WIDE_INT *, tree *, int);
 extern rtx_insn *asan_emit_allocas_unpoison (rtx, rtx, rtx_insn *);
@@ -188,6 +190,9 @@ extern hash_set<tree> *asan_handled_variables;
 static inline bool
 asan_intercepted_p (enum built_in_function fcode)
 {
+  if (memory_tagging_p ())
+    return false;
+
   return fcode == BUILT_IN_INDEX
 	 || fcode == BUILT_IN_MEMCHR
 	 || fcode == BUILT_IN_MEMCMP
@@ -216,7 +221,8 @@ asan_intercepted_p (enum built_in_function fcode)
 static inline bool
 asan_sanitize_use_after_scope (void)
 {
-  return (flag_sanitize_address_use_after_scope && asan_sanitize_stack_p ());
+  return (flag_sanitize_address_use_after_scope &&
+	  (asan_sanitize_stack_p () || memory_tagging_p ()));
 }
 
 /* Return true if DECL should be guarded on the stack.  */
diff --git a/gcc/asan.c b/gcc/asan.c
index 0e74e32ae6ca4e130b3f13abe110364b119def46..ae1f8a0d28e911c2ff30be8ea9f4001923983cb1 100644
--- a/gcc/asan.c
+++ b/gcc/asan.c
@@ -2177,7 +2177,13 @@ build_check_stmt (location_t loc, tree base, tree len,
   if (is_scalar_access)
     flags |= ASAN_CHECK_SCALAR_ACCESS;
 
-  g = gimple_build_call_internal (IFN_ASAN_CHECK, 4,
+  enum internal_fn fn;
+  if (memory_tagging_p ())
+    fn = IFN_HWASAN_CHECK;
+  else
+    fn = IFN_ASAN_CHECK;
+
+  g = gimple_build_call_internal (fn, 4,
 				  build_int_cst (integer_type_node, flags),
 				  base, len,
 				  build_int_cst (integer_type_node,
@@ -2201,10 +2207,13 @@ static void
 instrument_derefs (gimple_stmt_iterator *iter, tree t,
 		   location_t location, bool is_store)
 {
-  if (is_store && !ASAN_INSTRUMENT_WRITES)
-    return;
-  if (!is_store && !ASAN_INSTRUMENT_READS)
-    return;
+  if (! memory_tagging_p ())
+    {
+    if (is_store && !ASAN_INSTRUMENT_WRITES)
+      return;
+    if (!is_store && !ASAN_INSTRUMENT_READS)
+      return;
+    }
 
   tree type, base;
   HOST_WIDE_INT size_in_bytes;
@@ -2248,10 +2257,21 @@ instrument_derefs (gimple_stmt_iterator *iter, tree t,
       return;
     }
 
+  /* TODO Understand when this can happen.
+      What's the point of ignoring these parts?
+      I guess non-byte sizes would be awkward to instrument?
+      When would this occur?  */
   if (!multiple_p (bitpos, BITS_PER_UNIT)
       || maybe_ne (bitsize, size_in_bytes * BITS_PER_UNIT))
     return;
 
+  /* TODO What are we checking, and why are we not instrumenting that?
+      If the "object" is stored in a register then do nothing?
+      If the "object" is a register then do nothing?
+
+      The second one makes a lot of sense, but I can"t just assume that"s
+      what"s being checked.
+    */
   if (VAR_P (inner) && DECL_HARD_REGISTER (inner))
     return;
 
@@ -2264,7 +2284,8 @@ instrument_derefs (gimple_stmt_iterator *iter, tree t,
     {
       if (DECL_THREAD_LOCAL_P (inner))
 	return;
-      if (!ASAN_GLOBALS && is_global_var (inner))
+      if ((memory_tagging_p () || !ASAN_GLOBALS)
+	  && is_global_var (inner))
         return;
       if (!TREE_STATIC (inner))
 	{
@@ -2472,6 +2493,8 @@ maybe_instrument_assignment (gimple_stmt_iterator *iter)
 static bool
 maybe_instrument_call (gimple_stmt_iterator *iter)
 {
+  if (memory_tagging_p ())
+    return false;
   gimple *stmt = gsi_stmt (*iter);
   bool is_builtin = gimple_call_builtin_p (stmt, BUILT_IN_NORMAL);
 
@@ -3702,6 +3725,17 @@ make_pass_asan_O0 (gcc::context *ctxt)
   return new pass_asan_O0 (ctxt);
 }
 
+/*  HWASAN  */
+static unsigned int
+hwasan_instrument (void)
+{
+  if (shadow_ptr_types[0] == NULL_TREE)
+    asan_init_shadow_ptr_types ();
+  transform_statements ();
+  last_alloca_addr = NULL_TREE;
+  return 0;
+}
+
 void
 hwasan_record_base (rtx base)
 {
@@ -3909,4 +3943,176 @@ hwasan_finish_file (void)
   flag_sanitize |= SANITIZE_HWADDRESS;
 }
 
+bool
+hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
+{
+  // TODO For now only implementing the function when using calls.
+  // This is a little easier, and means I can rely on the library
+  // implementation while checking my instrumentation code for now.
+
+  gimple *g = gsi_stmt (*iter);
+  location_t loc = gimple_location (g);
+  bool recover_p = false;
+  (void)recover_p; // UNUSED for now (will be used to determine action)
+
+  HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (g, 0));
+  gcc_assert (flags < ASAN_CHECK_LAST);
+  bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
+  bool is_store = (flags & ASAN_CHECK_STORE) != 0;
+  bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
+
+  tree base = gimple_call_arg (g, 1);
+  tree len = gimple_call_arg (g, 2);
+
+  /* TODO align is unused for HWASAN_CHECK, but I pass the argument anyway
+   * because that way I need to write less code.  */
+  /* HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3));  */
+
+  unsigned HOST_WIDE_INT size_in_bytes
+    = is_scalar_access && tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
+  (void)size_in_bytes; // UNUSED for now (will be used to determine action)
+
+  gimple_stmt_iterator gsi = *iter;
+
+  if (!is_non_zero_len)
+    {
+      /* So, the length of the memory area to hwasan-protect is
+	 non-constant.  Let's guard the generated instrumentation code
+	 like:
+
+	 if (len != 0)
+	   {
+	     // hwasan instrumentation code goes here.
+	   }
+	 // falltrough instructions, starting with *ITER.  */
+
+      g = gimple_build_cond (NE_EXPR,
+			    len,
+			    build_int_cst (TREE_TYPE (len), 0),
+			    NULL_TREE, NULL_TREE);
+      gimple_set_location (g, loc);
+
+      basic_block then_bb, fallthrough_bb;
+      insert_if_then_before_iter (as_a <gcond *> (g), iter,
+				  /*then_more_likely_p=*/true,
+				  &then_bb, &fallthrough_bb);
+      /* Note that fallthrough_bb starts with the statement that was
+	pointed to by ITER.  */
+
+      /* The 'then block' of the 'if (len != 0) condition is where
+	we'll generate the hwasan instrumentation code now.  */
+      gsi = gsi_last_bb (then_bb);
+    }
+
+  /* Instrument using callbacks.  */
+  g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
+			   NOP_EXPR, base);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+  tree base_addr = gimple_assign_lhs (g);
+
+  /* TODO Here we only ever use the LOADN/STOREN functions for checking.
+     This means we always terminate the program on tag mismatch, always use
+     a function call instead of an inline check, and never have the nicer error
+     messages that come from size-specific checking.
+
+     This is much quicker to code for now, all other options will be
+     implemented later.  */
+  enum built_in_function fun_enum =
+    is_store ? BUILT_IN_HWASAN_STOREN : BUILT_IN_HWASAN_LOADN;
+  tree fun = builtin_decl_implicit (fun_enum);
+  g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
+			   NOP_EXPR, len);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+  tree sz_arg = gimple_assign_lhs (g);
+  g = gimple_build_call (fun, 2, base_addr, sz_arg);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+
+  gsi_remove (iter, true);
+  *iter = gsi;
+  return false;
+}
+
+bool
+gate_hwasan ()
+{
+  return memory_tagging_p ();
+}
+
+namespace {
+
+const pass_data pass_data_hwasan =
+{
+  GIMPLE_PASS, /* type */
+  "hwasan", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  TODO_update_ssa, /* todo_flags_finish */
+};
+
+class pass_hwasan : public gimple_opt_pass
+{
+public:
+  pass_hwasan (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_hwasan, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  opt_pass * clone () { return new pass_hwasan (m_ctxt); }
+  virtual bool gate (function *) { return gate_hwasan (); }
+  virtual unsigned int execute (function *) { return hwasan_instrument (); }
+
+}; // class pass_asan
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_hwasan (gcc::context *ctxt)
+{
+  return new pass_hwasan (ctxt);
+}
+
+namespace {
+
+const pass_data pass_data_hwasan_O0 =
+{
+  GIMPLE_PASS, /* type */
+  "hwasan_O0", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  TODO_update_ssa, /* todo_flags_finish */
+};
+
+class pass_hwasan_O0 : public gimple_opt_pass
+{
+public:
+  pass_hwasan_O0 (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_hwasan_O0, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  opt_pass * clone () { return new pass_hwasan_O0 (m_ctxt); }
+  virtual bool gate (function *) { return !optimize && gate_hwasan (); }
+  virtual unsigned int execute (function *) { return hwasan_instrument (); }
+
+}; // class pass_asan
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_hwasan_O0 (gcc::context *ctxt)
+{
+  return new pass_hwasan_O0 (ctxt);
+}
+
 #include "gt-asan.h"
diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 04081f36c4d31ecfba4099e50412345c67e1f58f..80f94f141bfd92e9f6af13a6df76f0c9ac053fdc 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -456,6 +456,12 @@ expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
 /* This should get expanded in the sanopt pass.  */
 
 static void
+expand_HWASAN_CHECK (internal_fn, gcall *)
+{
+  gcc_unreachable ();
+}
+
+static void
 expand_ASAN_CHECK (internal_fn, gcall *)
 {
   gcc_unreachable ();
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 016301a58d83d7128817824d7c7ef92825c7e03e..c683e5d8e5c607f18909bda4d97b58421cb7c2a4 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -288,6 +288,7 @@ DEF_INTERNAL_FN (UBSAN_PTR, ECF_LEAF | ECF_NOTHROW, ".R.")
 DEF_INTERNAL_FN (UBSAN_OBJECT_SIZE, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (ABNORMAL_DISPATCHER, ECF_NORETURN, NULL)
 DEF_INTERNAL_FN (BUILTIN_EXPECT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (HWASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
 DEF_INTERNAL_FN (ASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
 DEF_INTERNAL_FN (ASAN_MARK, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (ASAN_POISON, ECF_LEAF | ECF_NOTHROW | ECF_NOVOPS, NULL)
diff --git a/gcc/passes.def b/gcc/passes.def
index ad2efabd3853d8d20562f66f4c5bb34694ec80f2..11c9fb20b042d55a7d52da4feda633dc5cd3052a 100644
--- a/gcc/passes.def
+++ b/gcc/passes.def
@@ -246,6 +246,7 @@ along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_sink_code);
       NEXT_PASS (pass_sancov);
       NEXT_PASS (pass_asan);
+      NEXT_PASS (pass_hwasan);
       NEXT_PASS (pass_tsan);
       NEXT_PASS (pass_dce);
       /* Pass group that runs when 1) enabled, 2) there are loops
@@ -362,6 +363,7 @@ along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_dce);
       NEXT_PASS (pass_sancov);
       NEXT_PASS (pass_asan);
+      NEXT_PASS (pass_hwasan);
       NEXT_PASS (pass_tsan);
       /* ???  We do want some kind of loop invariant motion, but we possibly
          need to adjust LIM to be more friendly towards preserving accurate
@@ -387,6 +389,7 @@ along with GCC; see the file COPYING3.  If not see
   NEXT_PASS (pass_sancov_O0);
   NEXT_PASS (pass_lower_switch_O0);
   NEXT_PASS (pass_asan_O0);
+  NEXT_PASS (pass_hwasan_O0);
   NEXT_PASS (pass_tsan_O0);
   NEXT_PASS (pass_sanopt);
   NEXT_PASS (pass_cleanup_eh);
diff --git a/gcc/sanitizer.def b/gcc/sanitizer.def
index 7bd50715f24a2cb154b578e2abdea4e8fcdb2107..0edf349cc23e846608b89d54a1024b9d99de9c4d 100644
--- a/gcc/sanitizer.def
+++ b/gcc/sanitizer.def
@@ -183,6 +183,10 @@ DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_SUBTRACT, "__sanitizer_ptr_sub",
 /* Hardware Address Sanitizer.  */
 DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_INIT, "__hwasan_init",
 		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_LOADN, "__hwasan_loadN",
+		      BT_FN_VOID_PTR_PTRMODE, ATTR_TMPURE_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_STOREN, "__hwasan_storeN",
+		      BT_FN_VOID_PTR_PTRMODE, ATTR_TMPURE_NOTHROW_LEAF_LIST)
 DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_TAG_MEM, "__hwasan_tag_memory",
 		      BT_FN_VOID_PTR_UINT8_SIZE, ATTR_NOTHROW_LIST)
 
diff --git a/gcc/sanopt.c b/gcc/sanopt.c
index 5cb98e1b50e4e1644072bd18d74797c3cac43c3f..31270153f3cf56bfbad593830de1b9334e7f65d1 100644
--- a/gcc/sanopt.c
+++ b/gcc/sanopt.c
@@ -772,7 +772,8 @@ sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
   basic_block son;
   gimple_stmt_iterator gsi;
   sanopt_info *info = (sanopt_info *) bb->aux;
-  bool asan_check_optimize = (flag_sanitize & SANITIZE_ADDRESS) != 0;
+  bool asan_check_optimize =
+    ((flag_sanitize & SANITIZE_ADDRESS) != 0) || memory_tagging_p ();
 
   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
     {
@@ -802,6 +803,7 @@ sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
       if (asan_check_optimize
 	  && gimple_call_builtin_p (stmt, BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT))
 	{
+	  gcc_assert (!memory_tagging_p ());
 	  use_operand_p use;
 	  gimple *use_stmt;
 	  if (single_imm_use (gimple_vdef (stmt), &use, &use_stmt))
@@ -830,6 +832,7 @@ sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
 	  case IFN_UBSAN_PTR:
 	    remove = maybe_optimize_ubsan_ptr_ifn (ctx, stmt);
 	    break;
+	  case IFN_HWASAN_CHECK:
 	  case IFN_ASAN_CHECK:
 	    if (asan_check_optimize)
 	      remove = maybe_optimize_asan_check_ifn (ctx, stmt);
@@ -1262,10 +1265,11 @@ pass_sanopt::execute (function *fun)
   /* Try to remove redundant checks.  */
   if (optimize
       && (flag_sanitize
-	  & (SANITIZE_NULL | SANITIZE_ALIGNMENT
+	  & (SANITIZE_NULL | SANITIZE_ALIGNMENT | SANITIZE_HWADDRESS
 	     | SANITIZE_ADDRESS | SANITIZE_VPTR | SANITIZE_POINTER_OVERFLOW)))
     asan_num_accesses = sanopt_optimize (fun, &contains_asan_mark);
-  else if (flag_sanitize & SANITIZE_ADDRESS)
+  else if (flag_sanitize & SANITIZE_ADDRESS
+	   || memory_tagging_p ())
     {
       gimple_stmt_iterator gsi;
       FOR_EACH_BB_FN (bb, fun)
@@ -1285,7 +1289,7 @@ pass_sanopt::execute (function *fun)
       sanitize_asan_mark_poison ();
     }
 
-  if (asan_sanitize_stack_p ())
+  if (asan_sanitize_stack_p () || memory_tagging_p ())
     sanitize_rewrite_addressable_params (fun);
 
   bool use_calls = ASAN_INSTRUMENTATION_WITH_CALL_THRESHOLD < INT_MAX
@@ -1327,6 +1331,9 @@ pass_sanopt::execute (function *fun)
 		case IFN_UBSAN_VPTR:
 		  no_next = ubsan_expand_vptr_ifn (&gsi);
 		  break;
+		case IFN_HWASAN_CHECK:
+		  no_next = hwasan_expand_check_ifn (&gsi, use_calls);
+		  break;
 		case IFN_ASAN_CHECK:
 		  no_next = asan_expand_check_ifn (&gsi, use_calls);
 		  break;
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index 3a0b3805d24dbd50141d4145563861e4ae3768f3..01ebd03205e57ee3a63d3344da8098160c081002 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -341,6 +341,8 @@ extern void register_pass (opt_pass* pass, pass_positioning_ops pos,
 
 extern gimple_opt_pass *make_pass_asan (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_asan_O0 (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_hwasan (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_hwasan_O0 (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_tsan (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_tsan_O0 (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_sancov (gcc::context *ctxt);

Patch

diff --git a/gcc/asan.h b/gcc/asan.h
index c5492ce35980d0b26d4707f96482b69dc76a525a..68ea1b4afaf9195553251a987df33788421fa142 100644
--- a/gcc/asan.h
+++ b/gcc/asan.h
@@ -32,7 +32,9 @@  extern void hwasan_tag_init ();
 extern rtx hwasan_create_untagged_base (rtx);
 extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
 extern rtx_insn *hwasan_emit_uncolour_frame (rtx, rtx, rtx_insn *);
+extern bool hwasan_expand_check_ifn (gimple_stmt_iterator *, bool);
 extern bool memory_tagging_p (void);
+extern bool gate_hwasan (void);
 extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
 					     HOST_WIDE_INT *, tree *, int);
 extern rtx_insn *asan_emit_allocas_unpoison (rtx, rtx, rtx_insn *);
@@ -188,6 +190,9 @@  extern hash_set<tree> *asan_handled_variables;
 static inline bool
 asan_intercepted_p (enum built_in_function fcode)
 {
+  if (memory_tagging_p ())
+    return false;
+
   return fcode == BUILT_IN_INDEX
 	 || fcode == BUILT_IN_MEMCHR
 	 || fcode == BUILT_IN_MEMCMP
@@ -216,7 +221,8 @@  asan_intercepted_p (enum built_in_function fcode)
 static inline bool
 asan_sanitize_use_after_scope (void)
 {
-  return (flag_sanitize_address_use_after_scope && asan_sanitize_stack_p ());
+  return (flag_sanitize_address_use_after_scope &&
+	  (asan_sanitize_stack_p () || memory_tagging_p ()));
 }
 
 /* Return true if DECL should be guarded on the stack.  */
diff --git a/gcc/asan.c b/gcc/asan.c
index 0e74e32ae6ca4e130b3f13abe110364b119def46..ae1f8a0d28e911c2ff30be8ea9f4001923983cb1 100644
--- a/gcc/asan.c
+++ b/gcc/asan.c
@@ -2177,7 +2177,13 @@  build_check_stmt (location_t loc, tree base, tree len,
   if (is_scalar_access)
     flags |= ASAN_CHECK_SCALAR_ACCESS;
 
-  g = gimple_build_call_internal (IFN_ASAN_CHECK, 4,
+  enum internal_fn fn;
+  if (memory_tagging_p ())
+    fn = IFN_HWASAN_CHECK;
+  else
+    fn = IFN_ASAN_CHECK;
+
+  g = gimple_build_call_internal (fn, 4,
 				  build_int_cst (integer_type_node, flags),
 				  base, len,
 				  build_int_cst (integer_type_node,
@@ -2201,10 +2207,13 @@  static void
 instrument_derefs (gimple_stmt_iterator *iter, tree t,
 		   location_t location, bool is_store)
 {
-  if (is_store && !ASAN_INSTRUMENT_WRITES)
-    return;
-  if (!is_store && !ASAN_INSTRUMENT_READS)
-    return;
+  if (! memory_tagging_p ())
+    {
+    if (is_store && !ASAN_INSTRUMENT_WRITES)
+      return;
+    if (!is_store && !ASAN_INSTRUMENT_READS)
+      return;
+    }
 
   tree type, base;
   HOST_WIDE_INT size_in_bytes;
@@ -2248,10 +2257,21 @@  instrument_derefs (gimple_stmt_iterator *iter, tree t,
       return;
     }
 
+  /* TODO Understand when this can happen.
+      What's the point of ignoring these parts?
+      I guess non-byte sizes would be awkward to instrument?
+      When would this occur?  */
   if (!multiple_p (bitpos, BITS_PER_UNIT)
       || maybe_ne (bitsize, size_in_bytes * BITS_PER_UNIT))
     return;
 
+  /* TODO What are we checking, and why are we not instrumenting that?
+      If the "object" is stored in a register then do nothing?
+      If the "object" is a register then do nothing?
+
+      The second one makes a lot of sense, but I can"t just assume that"s
+      what"s being checked.
+    */
   if (VAR_P (inner) && DECL_HARD_REGISTER (inner))
     return;
 
@@ -2264,7 +2284,8 @@  instrument_derefs (gimple_stmt_iterator *iter, tree t,
     {
       if (DECL_THREAD_LOCAL_P (inner))
 	return;
-      if (!ASAN_GLOBALS && is_global_var (inner))
+      if ((memory_tagging_p () || !ASAN_GLOBALS)
+	  && is_global_var (inner))
         return;
       if (!TREE_STATIC (inner))
 	{
@@ -2472,6 +2493,8 @@  maybe_instrument_assignment (gimple_stmt_iterator *iter)
 static bool
 maybe_instrument_call (gimple_stmt_iterator *iter)
 {
+  if (memory_tagging_p ())
+    return false;
   gimple *stmt = gsi_stmt (*iter);
   bool is_builtin = gimple_call_builtin_p (stmt, BUILT_IN_NORMAL);
 
@@ -3702,6 +3725,17 @@  make_pass_asan_O0 (gcc::context *ctxt)
   return new pass_asan_O0 (ctxt);
 }
 
+/*  HWASAN  */
+static unsigned int
+hwasan_instrument (void)
+{
+  if (shadow_ptr_types[0] == NULL_TREE)
+    asan_init_shadow_ptr_types ();
+  transform_statements ();
+  last_alloca_addr = NULL_TREE;
+  return 0;
+}
+
 void
 hwasan_record_base (rtx base)
 {
@@ -3909,4 +3943,176 @@  hwasan_finish_file (void)
   flag_sanitize |= SANITIZE_HWADDRESS;
 }
 
+bool
+hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
+{
+  // TODO For now only implementing the function when using calls.
+  // This is a little easier, and means I can rely on the library
+  // implementation while checking my instrumentation code for now.
+
+  gimple *g = gsi_stmt (*iter);
+  location_t loc = gimple_location (g);
+  bool recover_p = false;
+  (void)recover_p; // UNUSED for now (will be used to determine action)
+
+  HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (g, 0));
+  gcc_assert (flags < ASAN_CHECK_LAST);
+  bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
+  bool is_store = (flags & ASAN_CHECK_STORE) != 0;
+  bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
+
+  tree base = gimple_call_arg (g, 1);
+  tree len = gimple_call_arg (g, 2);
+
+  /* TODO align is unused for HWASAN_CHECK, but I pass the argument anyway
+   * because that way I need to write less code.  */
+  /* HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3));  */
+
+  unsigned HOST_WIDE_INT size_in_bytes
+    = is_scalar_access && tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
+  (void)size_in_bytes; // UNUSED for now (will be used to determine action)
+
+  gimple_stmt_iterator gsi = *iter;
+
+  if (!is_non_zero_len)
+    {
+      /* So, the length of the memory area to hwasan-protect is
+	 non-constant.  Let's guard the generated instrumentation code
+	 like:
+
+	 if (len != 0)
+	   {
+	     // hwasan instrumentation code goes here.
+	   }
+	 // falltrough instructions, starting with *ITER.  */
+
+      g = gimple_build_cond (NE_EXPR,
+			    len,
+			    build_int_cst (TREE_TYPE (len), 0),
+			    NULL_TREE, NULL_TREE);
+      gimple_set_location (g, loc);
+
+      basic_block then_bb, fallthrough_bb;
+      insert_if_then_before_iter (as_a <gcond *> (g), iter,
+				  /*then_more_likely_p=*/true,
+				  &then_bb, &fallthrough_bb);
+      /* Note that fallthrough_bb starts with the statement that was
+	pointed to by ITER.  */
+
+      /* The 'then block' of the 'if (len != 0) condition is where
+	we'll generate the hwasan instrumentation code now.  */
+      gsi = gsi_last_bb (then_bb);
+    }
+
+  /* Instrument using callbacks.  */
+  g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
+			   NOP_EXPR, base);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+  tree base_addr = gimple_assign_lhs (g);
+
+  /* TODO Here we only ever use the LOADN/STOREN functions for checking.
+     This means we always terminate the program on tag mismatch, always use
+     a function call instead of an inline check, and never have the nicer error
+     messages that come from size-specific checking.
+
+     This is much quicker to code for now, all other options will be
+     implemented later.  */
+  enum built_in_function fun_enum =
+    is_store ? BUILT_IN_HWASAN_STOREN : BUILT_IN_HWASAN_LOADN;
+  tree fun = builtin_decl_implicit (fun_enum);
+  g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
+			   NOP_EXPR, len);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+  tree sz_arg = gimple_assign_lhs (g);
+  g = gimple_build_call (fun, 2, base_addr, sz_arg);
+  gimple_set_location (g, loc);
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+
+  gsi_remove (iter, true);
+  *iter = gsi;
+  return false;
+}
+
+bool
+gate_hwasan ()
+{
+  return memory_tagging_p ();
+}
+
+namespace {
+
+const pass_data pass_data_hwasan =
+{
+  GIMPLE_PASS, /* type */
+  "hwasan", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  TODO_update_ssa, /* todo_flags_finish */
+};
+
+class pass_hwasan : public gimple_opt_pass
+{
+public:
+  pass_hwasan (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_hwasan, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  opt_pass * clone () { return new pass_hwasan (m_ctxt); }
+  virtual bool gate (function *) { return gate_hwasan (); }
+  virtual unsigned int execute (function *) { return hwasan_instrument (); }
+
+}; // class pass_asan
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_hwasan (gcc::context *ctxt)
+{
+  return new pass_hwasan (ctxt);
+}
+
+namespace {
+
+const pass_data pass_data_hwasan_O0 =
+{
+  GIMPLE_PASS, /* type */
+  "hwasan_O0", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  TODO_update_ssa, /* todo_flags_finish */
+};
+
+class pass_hwasan_O0 : public gimple_opt_pass
+{
+public:
+  pass_hwasan_O0 (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_hwasan_O0, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  opt_pass * clone () { return new pass_hwasan_O0 (m_ctxt); }
+  virtual bool gate (function *) { return !optimize && gate_hwasan (); }
+  virtual unsigned int execute (function *) { return hwasan_instrument (); }
+
+}; // class pass_asan
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_hwasan_O0 (gcc::context *ctxt)
+{
+  return new pass_hwasan_O0 (ctxt);
+}
+
 #include "gt-asan.h"
diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 04081f36c4d31ecfba4099e50412345c67e1f58f..80f94f141bfd92e9f6af13a6df76f0c9ac053fdc 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -456,6 +456,12 @@  expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
 /* This should get expanded in the sanopt pass.  */
 
 static void
+expand_HWASAN_CHECK (internal_fn, gcall *)
+{
+  gcc_unreachable ();
+}
+
+static void
 expand_ASAN_CHECK (internal_fn, gcall *)
 {
   gcc_unreachable ();
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 016301a58d83d7128817824d7c7ef92825c7e03e..c683e5d8e5c607f18909bda4d97b58421cb7c2a4 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -288,6 +288,7 @@  DEF_INTERNAL_FN (UBSAN_PTR, ECF_LEAF | ECF_NOTHROW, ".R.")
 DEF_INTERNAL_FN (UBSAN_OBJECT_SIZE, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (ABNORMAL_DISPATCHER, ECF_NORETURN, NULL)
 DEF_INTERNAL_FN (BUILTIN_EXPECT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (HWASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
 DEF_INTERNAL_FN (ASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
 DEF_INTERNAL_FN (ASAN_MARK, ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (ASAN_POISON, ECF_LEAF | ECF_NOTHROW | ECF_NOVOPS, NULL)
diff --git a/gcc/passes.def b/gcc/passes.def
index ad2efabd3853d8d20562f66f4c5bb34694ec80f2..11c9fb20b042d55a7d52da4feda633dc5cd3052a 100644
--- a/gcc/passes.def
+++ b/gcc/passes.def
@@ -246,6 +246,7 @@  along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_sink_code);
       NEXT_PASS (pass_sancov);
       NEXT_PASS (pass_asan);
+      NEXT_PASS (pass_hwasan);
       NEXT_PASS (pass_tsan);
       NEXT_PASS (pass_dce);
       /* Pass group that runs when 1) enabled, 2) there are loops
@@ -362,6 +363,7 @@  along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_dce);
       NEXT_PASS (pass_sancov);
       NEXT_PASS (pass_asan);
+      NEXT_PASS (pass_hwasan);
       NEXT_PASS (pass_tsan);
       /* ???  We do want some kind of loop invariant motion, but we possibly
          need to adjust LIM to be more friendly towards preserving accurate
@@ -387,6 +389,7 @@  along with GCC; see the file COPYING3.  If not see
   NEXT_PASS (pass_sancov_O0);
   NEXT_PASS (pass_lower_switch_O0);
   NEXT_PASS (pass_asan_O0);
+  NEXT_PASS (pass_hwasan_O0);
   NEXT_PASS (pass_tsan_O0);
   NEXT_PASS (pass_sanopt);
   NEXT_PASS (pass_cleanup_eh);
diff --git a/gcc/sanitizer.def b/gcc/sanitizer.def
index 7bd50715f24a2cb154b578e2abdea4e8fcdb2107..0edf349cc23e846608b89d54a1024b9d99de9c4d 100644
--- a/gcc/sanitizer.def
+++ b/gcc/sanitizer.def
@@ -183,6 +183,10 @@  DEF_SANITIZER_BUILTIN(BUILT_IN_ASAN_POINTER_SUBTRACT, "__sanitizer_ptr_sub",
 /* Hardware Address Sanitizer.  */
 DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_INIT, "__hwasan_init",
 		      BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_LOADN, "__hwasan_loadN",
+		      BT_FN_VOID_PTR_PTRMODE, ATTR_TMPURE_NOTHROW_LEAF_LIST)
+DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_STOREN, "__hwasan_storeN",
+		      BT_FN_VOID_PTR_PTRMODE, ATTR_TMPURE_NOTHROW_LEAF_LIST)
 DEF_SANITIZER_BUILTIN(BUILT_IN_HWASAN_TAG_MEM, "__hwasan_tag_memory",
 		      BT_FN_VOID_PTR_UINT8_SIZE, ATTR_NOTHROW_LIST)
 
diff --git a/gcc/sanopt.c b/gcc/sanopt.c
index 5cb98e1b50e4e1644072bd18d74797c3cac43c3f..31270153f3cf56bfbad593830de1b9334e7f65d1 100644
--- a/gcc/sanopt.c
+++ b/gcc/sanopt.c
@@ -772,7 +772,8 @@  sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
   basic_block son;
   gimple_stmt_iterator gsi;
   sanopt_info *info = (sanopt_info *) bb->aux;
-  bool asan_check_optimize = (flag_sanitize & SANITIZE_ADDRESS) != 0;
+  bool asan_check_optimize =
+    ((flag_sanitize & SANITIZE_ADDRESS) != 0) || memory_tagging_p ();
 
   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
     {
@@ -802,6 +803,7 @@  sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
       if (asan_check_optimize
 	  && gimple_call_builtin_p (stmt, BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT))
 	{
+	  gcc_assert (!memory_tagging_p ());
 	  use_operand_p use;
 	  gimple *use_stmt;
 	  if (single_imm_use (gimple_vdef (stmt), &use, &use_stmt))
@@ -830,6 +832,7 @@  sanopt_optimize_walker (basic_block bb, struct sanopt_ctx *ctx)
 	  case IFN_UBSAN_PTR:
 	    remove = maybe_optimize_ubsan_ptr_ifn (ctx, stmt);
 	    break;
+	  case IFN_HWASAN_CHECK:
 	  case IFN_ASAN_CHECK:
 	    if (asan_check_optimize)
 	      remove = maybe_optimize_asan_check_ifn (ctx, stmt);
@@ -1262,10 +1265,11 @@  pass_sanopt::execute (function *fun)
   /* Try to remove redundant checks.  */
   if (optimize
       && (flag_sanitize
-	  & (SANITIZE_NULL | SANITIZE_ALIGNMENT
+	  & (SANITIZE_NULL | SANITIZE_ALIGNMENT | SANITIZE_HWADDRESS
 	     | SANITIZE_ADDRESS | SANITIZE_VPTR | SANITIZE_POINTER_OVERFLOW)))
     asan_num_accesses = sanopt_optimize (fun, &contains_asan_mark);
-  else if (flag_sanitize & SANITIZE_ADDRESS)
+  else if (flag_sanitize & SANITIZE_ADDRESS
+	   || memory_tagging_p ())
     {
       gimple_stmt_iterator gsi;
       FOR_EACH_BB_FN (bb, fun)
@@ -1285,7 +1289,7 @@  pass_sanopt::execute (function *fun)
       sanitize_asan_mark_poison ();
     }
 
-  if (asan_sanitize_stack_p ())
+  if (asan_sanitize_stack_p () || memory_tagging_p ())
     sanitize_rewrite_addressable_params (fun);
 
   bool use_calls = ASAN_INSTRUMENTATION_WITH_CALL_THRESHOLD < INT_MAX
@@ -1327,6 +1331,9 @@  pass_sanopt::execute (function *fun)
 		case IFN_UBSAN_VPTR:
 		  no_next = ubsan_expand_vptr_ifn (&gsi);
 		  break;
+		case IFN_HWASAN_CHECK:
+		  no_next = hwasan_expand_check_ifn (&gsi, use_calls);
+		  break;
 		case IFN_ASAN_CHECK:
 		  no_next = asan_expand_check_ifn (&gsi, use_calls);
 		  break;
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index 3a0b3805d24dbd50141d4145563861e4ae3768f3..01ebd03205e57ee3a63d3344da8098160c081002 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -341,6 +341,8 @@  extern void register_pass (opt_pass* pass, pass_positioning_ops pos,
 
 extern gimple_opt_pass *make_pass_asan (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_asan_O0 (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_hwasan (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_hwasan_O0 (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_tsan (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_tsan_O0 (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_sancov (gcc::context *ctxt);