[binutils-gdb] x86: refine when to trigger optimizations

[Jan Beulich]

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=72aea32839b5f4a2c8fa67e60451169b841941c0

commit 72aea32839b5f4a2c8fa67e60451169b841941c0
Author: Jan Beulich <jbeulich@suse.com>
Date:   Thu Jan 9 11:40:04 2020 +0100

    x86: refine when to trigger optimizations
    
    Checking just the base opcode without also checking this isn't a VEX
    encoding, and without there being other insn properties avoiding a match
    once respective VEX/XOP/EXEX-encoded insns would appear, is at least
    dangerous. Add respective checks. At the same time there's no real need
    to check the extension opcode to be None for the 0xA8 form - there's
    nothing it can be confused with, and non-VEX-and-alike forms also can't
    appear.

Diff:
---
 gas/ChangeLog        |  6 ++++++
 gas/config/tc-i386.c | 19 +++++++++----------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 26a0075..660b9a2 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,11 @@
 2020-01-09  Jan Beulich  <jbeulich@suse.com>
 
+	* config/tc-i386.c (optimize_encoding): Add
+	is_any_vex_encoding() invocations. Drop respective
+	i.tm.extension_opcode == None checks.
+
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
 	* config/tc-i386.c (md_assemble): Check RegRex is clear during
 	REX transformations. Correct comment indentation.
 
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 645fd38..2c087e4 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3990,13 +3990,13 @@ optimize_encoding (void)
   unsigned int j;
 
   if (optimize_for_space
+      && !is_any_vex_encoding (&i.tm)
       && i.reg_operands == 1
       && i.imm_operands == 1
       && !i.types[1].bitfield.byte
       && i.op[0].imms->X_op == O_constant
       && fits_in_imm7 (i.op[0].imms->X_add_number)
-      && ((i.tm.base_opcode == 0xa8
-	   && i.tm.extension_opcode == None)
+      && (i.tm.base_opcode == 0xa8
 	  || (i.tm.base_opcode == 0xf6
 	      && i.tm.extension_opcode == 0x0)))
     {
@@ -4022,6 +4022,7 @@ optimize_encoding (void)
 	}
     }
   else if (flag_code == CODE_64BIT
+	   && !is_any_vex_encoding (&i.tm)
 	   && ((i.types[1].bitfield.qword
 		&& i.reg_operands == 1
 		&& i.imm_operands == 1
@@ -4030,9 +4031,8 @@ optimize_encoding (void)
 		     && i.tm.extension_opcode == None
 		     && fits_in_unsigned_long (i.op[0].imms->X_add_number))
 		    || (fits_in_imm31 (i.op[0].imms->X_add_number)
-			&& (((i.tm.base_opcode == 0x24
-			      || i.tm.base_opcode == 0xa8)
-			     && i.tm.extension_opcode == None)
+			&& ((i.tm.base_opcode == 0x24
+			     || i.tm.base_opcode == 0xa8)
 			    || (i.tm.base_opcode == 0x80
 				&& i.tm.extension_opcode == 0x4)
 			    || ((i.tm.base_opcode == 0xf6
@@ -4044,13 +4044,11 @@ optimize_encoding (void)
 	       || (i.types[0].bitfield.qword
 		   && ((i.reg_operands == 2
 			&& i.op[0].regs == i.op[1].regs
-			&& ((i.tm.base_opcode == 0x30
-			     || i.tm.base_opcode == 0x28)
-			    && i.tm.extension_opcode == None))
+			&& (i.tm.base_opcode == 0x30
+			    || i.tm.base_opcode == 0x28))
 		       || (i.reg_operands == 1
 			   && i.operands == 1
-			   && i.tm.base_opcode == 0x30
-			   && i.tm.extension_opcode == None)))))
+			   && i.tm.base_opcode == 0x30)))))
     {
       /* Optimize: -O:
 	   andq $imm31, %r64   -> andl $imm31, %r32
@@ -4091,6 +4089,7 @@ optimize_encoding (void)
     }
   else if (optimize > 1
 	   && !optimize_for_space
+	   && !is_any_vex_encoding (&i.tm)
 	   && i.reg_operands == 2
 	   && i.op[0].regs == i.op[1].regs
 	   && ((i.tm.base_opcode & ~(Opcode_D | 1)) == 0x8