ubsan: alpha-vms: shift exponent 536874240 is too large

Message ID 20200624011801.GR21885@bubble.grove.modra.org
State New
Headers show
Series
  • ubsan: alpha-vms: shift exponent 536874240 is too large
Related show

Commit Message

David Faust via Binutils June 24, 2020, 1:18 a.m.
C_OPR_ASH is supposed to be an arithmetic shift.  By the look of it,
this operator implemented logical shifts since the original binutils
support was added.  This patch corrects that and avoids some nonsense
ubsan complaints.  I chose to implement infinite precision shifts
rather than masking shift counts to the word size as the spec I had is
silent on what is supposed to happen with overlarge shift counts.

	* vms-alpha.c (_bfd_vms_slurp_etir <ETIR__C_OPR_ASH>): Implement
	shifts without undefined behaviour.


-- 
Alan Modra
Australia Development Lab, IBM

Patch

diff --git a/bfd/vms-alpha.c b/bfd/vms-alpha.c
index e31a9e4ca1..5bf32d6780 100644
--- a/bfd/vms-alpha.c
+++ b/bfd/vms-alpha.c
@@ -34,6 +34,7 @@ 
 */
 
 #include "sysdep.h"
+#include <limits.h>
 #include "bfd.h"
 #include "bfdlink.h"
 #include "libbfd.h"
@@ -71,6 +72,9 @@ 
 
 
 #define MIN(a,b) ((a) < (b) ? (a) : (b))
+#ifndef CHAR_BIT
+#define CHAR_BIT 8
+#endif
 
 /* The r_type field in a reloc is one of the following values.  */
 #define ALPHA_R_IGNORE		0
@@ -2520,10 +2524,27 @@  _bfd_vms_slurp_etir (bfd *abfd, struct bfd_link_info *info)
 				  _bfd_vms_etir_name (cmd));
 	      return FALSE;
 	    }
-	  if ((int)op2 < 0)		/* Shift right.  */
-	    op1 >>= -(int)op2;
-	  else			/* Shift left.  */
-	    op1 <<= (int)op2;
+	  if ((bfd_signed_vma) op2 < 0)
+	    {
+	      /* Shift right.  */
+	      bfd_vma sign;
+	      op2 = -op2;
+	      if (op2 >= CHAR_BIT * sizeof (op1))
+		op2 = CHAR_BIT * sizeof (op1) - 1;
+	      /* op1 = (bfd_signed_vma) op1 >> op2; */
+	      sign = op1 & ((bfd_vma) 1 << (CHAR_BIT * sizeof (op1) - 1));
+	      op1 >>= op2;
+	      sign >>= op2;
+	      op1 = (op1 ^ sign) - sign;
+	    }
+	  else
+	    {
+	      /* Shift left.  */
+	      if (op2 >= CHAR_BIT * sizeof (op1))
+		op1 = 0;
+	      else
+		op1 <<= op2;
+	    }
 	  if (!_bfd_vms_push (abfd, op1, RELC_NONE)) /* FIXME: sym.  */
 	    return FALSE;
 	  break;