[binutils-gdb] x86/APX: squash REX prefix when REX2 is being emitted

[Jan Beulich]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=d0a9378d985b3fe7b52369ec7c64088f3171fdc9

commit d0a9378d985b3fe7b52369ec7c64088f3171fdc9
Author: Jan Beulich <jbeulich@suse.com>
Date:   Wed Oct 30 12:12:24 2024 +0100

    x86/APX: squash REX prefix when REX2 is being emitted
    
    We should not (silently) emit a REX prefix ahead of a REX2-encoded insn;
    such encodings are illegal. Best we can do is fold the REX bits into the
    REX2 prefix, and then zap the REX one from i.prefix[].

Diff:
---
 gas/config/tc-i386.c                     | 4 +++-
 gas/testsuite/gas/i386/x86-64-apx-rex2.d | 4 ++++
 gas/testsuite/gas/i386/x86-64-apx-rex2.s | 6 ++++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 9f68a2fb412..f6a8c30b379 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -4527,7 +4527,8 @@ build_rex2_prefix (void)
   i.vex.bytes[0] = 0xd5;
   /* For the W R X B bits, the variables of rex prefix will be reused.  */
   i.vex.bytes[1] = ((i.tm.opcode_space << 7)
-		    | (i.rex2 << 4) | i.rex);
+		    | (i.rex2 << 4)
+		    | ((i.rex | i.prefix[REX_PREFIX]) & 0xf));
 }
 
 /* Build the EVEX prefix (4-byte) for evex insn
@@ -4676,6 +4677,7 @@ static void establish_rex (void)
       build_rex2_prefix ();
       /* The individual REX.RXBW bits got consumed.  */
       i.rex &= REX_OPCODE;
+      i.prefix[REX_PREFIX] = 0;
     }
   else if (i.rex != 0)
     add_prefix (REX_OPCODE | i.rex);
diff --git a/gas/testsuite/gas/i386/x86-64-apx-rex2.d b/gas/testsuite/gas/i386/x86-64-apx-rex2.d
index e3cd534da11..0783f9b979c 100644
--- a/gas/testsuite/gas/i386/x86-64-apx-rex2.d
+++ b/gas/testsuite/gas/i386/x86-64-apx-rex2.d
@@ -80,4 +80,8 @@ Disassembly of section .text:
 [	 ]*[a-f0-9]+:[	 ]*d5 76 8d 7c 20 01    	lea    0x1\(%r16,%r28,1\),%r31d
 [	 ]*[a-f0-9]+:[	 ]*d5 12 8d 84 04 81 00 00 00 	lea    0x81\(%r20,%r8,1\),%eax
 [	 ]*[a-f0-9]+:[	 ]*d5 57 8d bc 04 81 00 00 00 	lea    0x81\(%r28,%r8,1\),%r31d
+[	 ]*[a-f0-9]+:[	 ]*d5 14 f7 14 24       	\{rex2 0x14\} notl \(%r20\)
+[	 ]*[a-f0-9]+:[	 ]*d5 12 f7 14 24       	notl   \(%r20,%r12,1\)
+[	 ]*[a-f0-9]+:[	 ]*d5 11 f7 14 24       	notl   \(%r28\)
+[	 ]*[a-f0-9]+:[	 ]*d5 18 f7 14 24       	notq   \(%r20\)
 #pass
diff --git a/gas/testsuite/gas/i386/x86-64-apx-rex2.s b/gas/testsuite/gas/i386/x86-64-apx-rex2.s
index eaaaaa77dd7..5102dc37c27 100644
--- a/gas/testsuite/gas/i386/x86-64-apx-rex2.s
+++ b/gas/testsuite/gas/i386/x86-64-apx-rex2.s
@@ -83,3 +83,9 @@ _start:
          leal	1(%r16, %r28), %r31d
          leal	129(%r20, %r8), %eax
          leal	129(%r28, %r8), %r31d
+
+## explicit REX prefix
+         rex.r notl (%r20)
+         rex.x notl (%r20)
+         rex.b notl (%r20)
+         rex.w not (%r20)