[PATCH 5/5] x86: consistently print prefixes explicitly which are invalid with VEX etc

[Jan Beulich]

All of data size, rep, lock, and rex prefixes are invalid with VEX- and
alike encoded insns. Make sure they get printed explicitly in all cases,
to signal the anomaly. With this, do away with "rex_ignored" - if there
is a rex prefix, we want to print it anyway for VEX etc (and there's
nothing "ignored" about it in the first place - such an instruction will
raise #UD).

gas/
2020-06-XX  Jan Beulich  <jbeulich@suse.com>

	* testsuite/gas/i386/prefix.s: Add bogus prefix-with-VEX/EVEX
	encoding tests.
	* testsuite/gas/i386/prefix.d: Adjust expectations.

opcodes/
2020-06-XX  Jan Beulich  <jbeulich@suse.com>

	* i386-dis.c (rex_ignored): Delete.
	(ckprefix): Drop rex_ignored initialization.
	(get_valid_dis386): Drop setting of rex_ignored.
	(print_insn): Drop checking of rex_ignored. Don't record data
	size prefix as used with VEX-and-alike encodings.

--- a/gas/testsuite/gas/i386/prefix.d
+++ b/gas/testsuite/gas/i386/prefix.d
@@ -86,8 +86,11 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:	ff cc                	dec    %esp
 [ 	]*[a-f0-9]+:	62 f1 ff 08 28       	\(bad\) *
 [ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	66 c5 f8 28 c0       	data16 vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f3 c4 e1 78 28 c0    	repz vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f2 c5 f8 28 c0       	repnz vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f0 62 f1 7c 08 28 c0 	lock vmovaps %xmm0,%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
 [ 	]*[a-f0-9]+:	62 f1 ff 18 e6 40 04 	vcvtpd2dq 0x20\(%eax\)\{1to2\},%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
-	...
 #pass
--- a/gas/testsuite/gas/i386/prefix.s
+++ b/gas/testsuite/gas/i386/prefix.s
@@ -451,6 +451,11 @@
 
 	int $3
 
+	.byte 0x66; vmovaps %xmm0, %xmm0
+	repz; {vex3} vmovaps %xmm0, %xmm0
+	repnz; vmovaps %xmm0, %xmm0
+	lock; {evex} vmovaps %xmm0, %xmm0
+
 	vcvtpd2dqx 0x20(%eax),%xmm0
 	vcvtpd2dq 0x20(%eax){1to2},%xmm0
 	vcvtpd2dqx 0x20(%eax),%xmm0
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -153,8 +153,6 @@ static int prefixes;
 static int rex;
 /* Bits of REX we've already used.  */
 static int rex_used;
-/* REX bits in original REX prefix ignored.  */
-static int rex_ignored;
 /* Mark parts used in the REX prefix.  When we are testing for
    empty prefix (for 8bit register REX extension), just mask it
    out.  Otherwise test for REX bit is excuse for existence of REX
@@ -11057,7 +11055,6 @@ ckprefix (void)
 {
   int newrex, i, length;
   rex = 0;
-  rex_ignored = 0;
   prefixes = 0;
   used_prefixes = 0;
   rex_used = 0;
@@ -11517,8 +11514,6 @@ get_valid_dis386 (const struct dis386 *d
 
     case USE_XOP_8F_TABLE:
       FETCH_DATA (info, codep + 3);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = ~(*codep >> 5) & 0x7;
 
       /* VEX_TABLE_INDEX is the mmmmm part of the XOP byte 1 "RCB.mmmmm".  */
@@ -11580,8 +11575,6 @@ get_valid_dis386 (const struct dis386 *d
     case USE_VEX_C4_TABLE:
       /* VEX prefix.  */
       FETCH_DATA (info, codep + 3);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = ~(*codep >> 5) & 0x7;
       switch ((*codep & 0x1f))
 	{
@@ -11647,8 +11640,6 @@ get_valid_dis386 (const struct dis386 *d
     case USE_VEX_C5_TABLE:
       /* VEX prefix.  */
       FETCH_DATA (info, codep + 2);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = (*codep & 0x80) ? 0 : REX_R;
 
       /* For the 2-byte VEX prefix in 32-bit mode, the highest bit in
@@ -11697,8 +11688,6 @@ get_valid_dis386 (const struct dis386 *d
       /* EVEX prefix.  */
       vex.evex = 1;
       FETCH_DATA (info, codep + 4);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       /* The first byte after 0x62.  */
       rex = ~(*codep >> 5) & 0x7;
       vex.r = *codep & 0x10;
@@ -12179,7 +12168,7 @@ print_insn (bfd_vma pc, disassemble_info
     }
 
   /* Check if the REX prefix is used.  */
-  if (rex_ignored == 0 && (rex ^ rex_used) == 0 && last_rex_prefix >= 0)
+  if ((rex ^ rex_used) == 0 && !need_vex && last_rex_prefix >= 0)
     all_prefixes[last_rex_prefix] = 0;
 
   /* Check if the SEG prefix is used.  */
@@ -12195,7 +12184,8 @@ print_insn (bfd_vma pc, disassemble_info
 
   /* Check if the DATA prefix is used.  */
   if ((prefixes & PREFIX_DATA) != 0
-      && (used_prefixes & PREFIX_DATA) != 0)
+      && (used_prefixes & PREFIX_DATA) != 0
+      && !need_vex)
     all_prefixes[last_data_prefix] = 0;
 
   /* Print the extra prefixes.  */