[PATCH 4/9] x86-64: REX prefix is invalid with VEX etc

[Jan Beulich]

Just like for the data size prefix (see commit 7a8655d2bbdc ["x86: don't
abort() upon DATA16 prefix on (E)VEX encoded insn"]), any form of REX
prefix is invalid with VEX/XOP/EVEX.

gas/
2020-06-XX  Jan Beulich  <jbeulich@suse.com>

	* config/tc-i386.c (md_assemble): Also reject explicit REX
	prefixes with VEX and alike encoded insns. Zap consumed bits
	from i.rex.
	(output_insn): Don't ignore REX prefix for VEX and alike
	encodings; abort() instead if encountered.
	* testsuite/gas/i386/x86-64-pseudos.s: Move REX-with-VEX cases
	...
	* testsuite/gas/i386/x86-64-pseudos-bad.s: ... here.
	* testsuite/gas/i386/x86-64-pseudos.d,
	testsuite/gas/i386/x86-64-pseudos-bad.l: Adjust expectations.

--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -4892,10 +4892,20 @@ md_assemble (char *line)
 	  return;
 	}
 
+      /* Check for explicit REX prefix.  */
+      if (i.prefix[REX_PREFIX] || i.rex_encoding)
+	{
+	  as_bad (_("REX prefix invalid with `%s'"), i.tm.name);
+	  return;
+	}
+
       if (i.tm.opcode_modifier.vex)
 	build_vex_prefix (t);
       else
 	build_evex_prefix ();
+
+      /* The individual REX.RXBW bits got consumed.  */
+      i.rex &= REX_OPCODE;
     }
 
   /* Handle conversion of 'int $3' --> special int3 insn.  XOP or FMA4
@@ -9274,9 +9284,6 @@ output_insn (void)
 	    if (*q)
 	      switch (j)
 		{
-		case REX_PREFIX:
-		  /* REX byte is encoded in VEX prefix.  */
-		  break;
 		case SEG_PREFIX:
 		case ADDR_PREFIX:
 		  FRAG_APPEND_1_CHAR (*q);
--- a/gas/testsuite/gas/i386/x86-64-pseudos-bad.l
+++ b/gas/testsuite/gas/i386/x86-64-pseudos-bad.l
@@ -1,3 +1,6 @@
 .*: Assembler messages:
 .*:3: Error: .*`vmovaps'.*
 .*:4: Error: .*`vmovaps'.*
+.*:5: Error: .*`vmovaps'.*
+.*:6: Error: .*`vmovaps'.*
+.*:7: Error: .*`rorx'.*
--- a/gas/testsuite/gas/i386/x86-64-pseudos-bad.s
+++ b/gas/testsuite/gas/i386/x86-64-pseudos-bad.s
@@ -2,3 +2,6 @@
 pseudos:
 	{vex} vmovaps %xmm0, %xmm30
 	{vex3} vmovaps %xmm30, %xmm0
+	{rex} vmovaps %xmm7,%xmm2
+	{rex} vmovaps %xmm17,%xmm2
+	{rex} rorx $7,%eax,%ebx
--- a/gas/testsuite/gas/i386/x86-64-pseudos.d
+++ b/gas/testsuite/gas/i386/x86-64-pseudos.d
@@ -310,9 +310,6 @@ Disassembly of section .text:
  +[a-f0-9]+:	41 0f 28 10          	movaps \(%r8\),%xmm2
  +[a-f0-9]+:	40 0f 38 01 01       	rex phaddw \(%rcx\),%mm0
  +[a-f0-9]+:	41 0f 38 01 00       	phaddw \(%r8\),%mm0
- +[a-f0-9]+:	c5 f8 28 d7          	vmovaps %xmm7,%xmm2
- +[a-f0-9]+:	62 b1 7c 08 28 d1    	vmovaps %xmm17,%xmm2
- +[a-f0-9]+:	c4 e3 7b f0 d8 07    	rorx   \$0x7,%eax,%ebx
  +[a-f0-9]+:	c4 e1 78 28 d7       	vmovaps %xmm7,%xmm2
  +[a-f0-9]+:	c4 e1 78 28 d7       	vmovaps %xmm7,%xmm2
  +[a-f0-9]+:	c4 e1 78 29 fa       	vmovaps %xmm7,%xmm2
@@ -351,7 +348,4 @@ Disassembly of section .text:
  +[a-f0-9]+:	41 0f 28 10          	movaps \(%r8\),%xmm2
  +[a-f0-9]+:	40 0f 38 01 01       	rex phaddw \(%rcx\),%mm0
  +[a-f0-9]+:	41 0f 38 01 00       	phaddw \(%r8\),%mm0
- +[a-f0-9]+:	c5 f8 28 d7          	vmovaps %xmm7,%xmm2
- +[a-f0-9]+:	62 b1 7c 08 28 d1    	vmovaps %xmm17,%xmm2
- +[a-f0-9]+:	c4 e3 7b f0 d8 07    	rorx   \$0x7,%eax,%ebx
 #pass
--- a/gas/testsuite/gas/i386/x86-64-pseudos.s
+++ b/gas/testsuite/gas/i386/x86-64-pseudos.s
@@ -314,9 +314,6 @@ _start:
 	{rex} movaps (%r8),%xmm2
 	{rex} phaddw (%rcx),%mm0
 	{rex} phaddw (%r8),%mm0
-	{rex} vmovaps %xmm7,%xmm2
-	{rex} vmovaps %xmm17,%xmm2
-	{rex} rorx $7,%eax,%ebx
 
 	.intel_syntax noprefix
 	{vex3} vmovaps xmm2,xmm7
@@ -357,6 +354,3 @@ _start:
 	{rex} movaps xmm2,XMMWORD PTR [r8]
 	{rex} phaddw mm0,QWORD PTR [rcx]
 	{rex} phaddw mm0,QWORD PTR [r8]
-	{rex} vmovaps xmm2,xmm7
-	{rex} vmovaps xmm2,xmm17
-	{rex} rorx ebx,eax,0x7