[2/5] x86: correct decoding of packed-FP-only AVX encodings

Message ID d6ca20eb-c8bc-4980-34ce-d34016c15837@suse.com
State New
Headers show
Series
  • x86: (mainly) insn prefix decoding adjustments
Related show

Commit Message

Jan Beulich June 8, 2020, 7:45 a.m.
Various AVX insns utilizing the X macro fail to reject F3/F2 embedded
prefix encodings. As the PREFIX_OPCODE attribute wasn't used by any
non-legacy-encoded insns so far, re-use it to achieve the intended
effect.

gas/
2020-06-XX  Jan Beulich  <jbeulich@suse.com>

	* testsuite/gas/i386/prefix.s: Add bogus REP / EVEX.W prefix
	with VEX/EVEX encoding tests.
	* testsuite/gas/i386/prefix.d: Adjust expectations.

opcodes/
2020-06-XX  Jan Beulich  <jbeulich@suse.com>

	* i386-dis.c (vex_table): Use PREFIX_OPCODE for vunpcklpX,
	vunpckhpX, vmovapX, vandpX, vandnpX, vorpX, vxorpX and vshufpX.
	(vex_len_table) : Likewise for vmovlpX, vmovhpX, vmovntpX, and
	vmovmskpX.
	(print_insn): Drop pointless check against bad_opcode. Split
	prefix validation into legacy and VEX-and-alike parts.
	(putop): Re-work 'X' macro handling.

---
This could in principle be used to significantly decrease table sizes
by making AVX extensions of originally MMX insns (encoded with an
embedded 66 prefix), by introducing a suitable macro and avoiding the
need for the various respective PREFIX_* enumerators / indirections.

Patch

--- a/gas/testsuite/gas/i386/prefix.d
+++ b/gas/testsuite/gas/i386/prefix.d
@@ -72,6 +72,20 @@  Disassembly of section .text:
 [ 	]*[a-f0-9]+:	90                   	nop
 [ 	]*[a-f0-9]+:	f2 0f c7             	\(bad\)  
 [ 	]*[a-f0-9]+:	f0 90                	lock nop
+[ 	]*[a-f0-9]+:	f3 0f 28             	repz \(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	c5 fa 28             	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	c4 e1 7b 28          	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 fc 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 7e 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 7d 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 ff 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
 [ 	]*[a-f0-9]+:	62 f1 ff 18 e6 40 04 	vcvtpd2dq 0x20\(%eax\)\{1to2\},%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
--- a/gas/testsuite/gas/i386/prefix.s
+++ b/gas/testsuite/gas/i386/prefix.s
@@ -391,6 +391,66 @@ 
 
 	nop
 
+	repz; movaps %xmm7, %xmm7
+	int $3
+
+# "repz" vmovaps %xmm7, %xmm7
+	.byte 0xc5
+	.byte 0xfa
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repnz" {vex3} vmovaps %xmm7, %xmm7
+	.byte 0xc4
+	.byte 0xe1
+	.byte 0x7b
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "EVEX.W1" vmovaps %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0xfc
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repz" vmovaps %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0x7e
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "EVEX.W0" vmovapd %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0x7d
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repnz" vmovapd %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0xff
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
 	vcvtpd2dqx 0x20(%eax),%xmm0
 	vcvtpd2dq 0x20(%eax){1to2},%xmm0
 	vcvtpd2dqx 0x20(%eax),%xmm0
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -8485,8 +8485,8 @@  static const struct dis386 vex_table[][2
     { PREFIX_TABLE (PREFIX_VEX_0F11) },
     { PREFIX_TABLE (PREFIX_VEX_0F12) },
     { MOD_TABLE (MOD_VEX_0F13) },
-    { "vunpcklpX",	{ XM, Vex, EXx }, 0 },
-    { "vunpckhpX",	{ XM, Vex, EXx }, 0 },
+    { "vunpcklpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vunpckhpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F16) },
     { MOD_TABLE (MOD_VEX_0F17) },
     /* 18 */
@@ -8508,8 +8508,8 @@  static const struct dis386 vex_table[][2
     { Bad_Opcode },
     { Bad_Opcode },
     /* 28 */
-    { "vmovapX",	{ XM, EXx }, 0 },
-    { "vmovapX",	{ EXxS, XM }, 0 },
+    { "vmovapX",	{ XM, EXx }, PREFIX_OPCODE },
+    { "vmovapX",	{ EXxS, XM }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F2A) },
     { MOD_TABLE (MOD_VEX_0F2B) },
     { PREFIX_TABLE (PREFIX_VEX_0F2C) },
@@ -8557,10 +8557,10 @@  static const struct dis386 vex_table[][2
     { PREFIX_TABLE (PREFIX_VEX_0F51) },
     { PREFIX_TABLE (PREFIX_VEX_0F52) },
     { PREFIX_TABLE (PREFIX_VEX_0F53) },
-    { "vandpX",		{ XM, Vex, EXx }, 0 },
-    { "vandnpX",	{ XM, Vex, EXx }, 0 },
-    { "vorpX",		{ XM, Vex, EXx }, 0 },
-    { "vxorpX",		{ XM, Vex, EXx }, 0 },
+    { "vandpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vandnpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vxorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
     /* 58 */
     { PREFIX_TABLE (PREFIX_VEX_0F58) },
     { PREFIX_TABLE (PREFIX_VEX_0F59) },
@@ -8685,7 +8685,7 @@  static const struct dis386 vex_table[][2
     { Bad_Opcode },
     { PREFIX_TABLE (PREFIX_VEX_0FC4) },
     { PREFIX_TABLE (PREFIX_VEX_0FC5) },
-    { "vshufpX",	{ XM, Vex, EXx, Ib }, 0 },
+    { "vshufpX",	{ XM, Vex, EXx, Ib }, PREFIX_OPCODE },
     { Bad_Opcode },
     /* c8 */
     { Bad_Opcode },
@@ -9355,7 +9355,7 @@  static const struct dis386 vex_len_table
 
   /* VEX_LEN_0F13_M_0 */
   {
-    { "vmovlpX",	{ EXq, XM }, 0 },
+    { "vmovlpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F16_P_0_M_0 */
@@ -9375,7 +9375,7 @@  static const struct dis386 vex_len_table
 
   /* VEX_LEN_0F17_M_0 */
   {
-    { "vmovhpX",	{ EXq, XM }, 0 },
+    { "vmovhpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F41_P_0 */
@@ -10592,7 +10592,7 @@  static const struct dis386 mod_table[][2
   },
   {
     /* MOD_VEX_0F2B */
-    { "vmovntpX",	{ Mx, XM }, 0 },
+    { "vmovntpX",	{ Mx, XM }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_W_0_0F41_P_0_LEN_1 */
@@ -10752,7 +10752,7 @@  static const struct dis386 mod_table[][2
   {
     /* MOD_VEX_0F50 */
     { Bad_Opcode },
-    { "vmovmskpX",	{ Gdq, XS }, 0 },
+    { "vmovmskpX",	{ Gdq, XS }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_0F71_REG_2 */
@@ -12266,14 +12266,18 @@  print_insn (bfd_vma pc, disassemble_info
      PREFIX_REPZ/PREFIX_REPNZ fix, we check the PREFIX_DATA prefix
      separately.  */
   if (dp->prefix_requirement == PREFIX_OPCODE
-      && dp != &bad_opcode
-      && (((prefixes
-	    & (PREFIX_REPZ | PREFIX_REPNZ)) != 0
+      && (((need_vex
+	    ? vex.prefix == REPE_PREFIX_OPCODE
+	      || vex.prefix == REPNE_PREFIX_OPCODE
+	    : (prefixes
+	       & (PREFIX_REPZ | PREFIX_REPNZ)) != 0)
 	   && (used_prefixes
 	       & (PREFIX_REPZ | PREFIX_REPNZ)) == 0)
-	  || ((((prefixes
-		 & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
-		== PREFIX_DATA)
+	  || (((need_vex
+		? vex.prefix == DATA_PREFIX_OPCODE
+		: ((prefixes
+		    & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
+		   == PREFIX_DATA))
 	       && (used_prefixes & PREFIX_DATA) == 0))))
     {
       (*info->fprintf_func) (info->stream, "(bad)");
@@ -13230,21 +13234,15 @@  putop (const char *in_template, int size
 	      SAVE_LAST (*p);
 	      break;
 	    }
-	  if (need_vex && vex.prefix)
+	  if (need_vex
+	      ? vex.prefix == DATA_PREFIX_OPCODE
+	      : prefixes & PREFIX_DATA)
 	    {
-	      if (vex.prefix == DATA_PREFIX_OPCODE)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
+	      *obufp++ = 'd';
+	      used_prefixes |= PREFIX_DATA;
 	    }
 	  else
-	    {
-	      if (prefixes & PREFIX_DATA)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
-	      used_prefixes |= (prefixes & PREFIX_DATA);
-	    }
+	    *obufp++ = 's';
 	  break;
 	case 'Y':
 	  if (l == 0 && len == 1)