[PATCH v2] x86: SYSENTER/SYSEXIT are unavailable in 64-bit mode on AMD

[Jan Beulich <jbeulich at suse dot com>]

The disassembler change is such that in default mode we'd disassemble
the insns (for there not ebing any conflicts), but when AMD64 mode was
explicitly requested, we'd show them as "(bad)".

gas/
2019-11-XX  Jan Beulich  <jbeulich@suse.com>

	* testsuite/gas/i386/x86-64-sysenter-amd.s,
	testsuite/gas/i386/x86-64-sysenter-amd.d,
	testsuite/gas/i386/x86-64-sysenter-amd.l,
	testsuite/gas/i386/x86-64-sysenter-intel.d,
	testsuite/gas/i386/x86-64-sysenter-mixed.d: New.
	* testsuite/gas/i386/i386.exp: Run new tests.

opcodes/
2019-11-XX  Jan Beulich  <jbeulich@suse.com>

	* i386-dis.c (SEP_Fixup): New.
	(SEP): Define.
	(dis386_twobyte): Use it for sysenter/sysexit.
	(enum x86_64_isa): Change amd64 enumerator to value 1.
	(OP_J): Compare isa64 against intel64 instead of amd64.
	* i386-opc.tbl (sysenter, sysexit): Split into AMD64 and Intel64
	forms.
	* i386-tbl.h: Re-generate.
---
v2: Accompany by disassembler change.

--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -702,6 +702,10 @@ if [expr ([istarget "i*86-*-*"] || [ista
     run_dump_test "x86-64-nops-5"
     run_dump_test "x86-64-nops-5-k8"
     run_dump_test "x86-64-nops-7"
+    run_dump_test "x86-64-sysenter-intel"
+    run_dump_test "x86-64-sysenter-mixed"
+    run_dump_test "x86-64-sysenter-amd"
+    run_list_test "x86-64-sysenter-amd"
     run_dump_test "noreg64"
     run_list_test "cvtsi2sX"
     run_dump_test "x86-64-sse4_1"
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-sysenter-amd.d
@@ -0,0 +1,14 @@
+#as: -mintel64
+#objdump: -dw -Mamd64
+#name: x86-64 sysenter (Intel64/AMD64)
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+[ 	]*[a-f0-9]+:[ 	]+0f 34[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 35[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 34[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 35[ 	]+\(bad\)[ 	]*
+#pass
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-sysenter-amd.l
@@ -0,0 +1,5 @@
+.*: Assembler messages:
+.*:2: Error: .*
+.*:3: Error: .*
+.*:6: Error: .*
+.*:7: Error: .*
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-sysenter-amd.s
@@ -0,0 +1,7 @@
+	.text
+	sysenter
+	sysexit
+
+	.intel_syntax noprefix
+	sysenter
+	sysexit
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-sysenter-intel.d
@@ -0,0 +1,15 @@
+#as: -mintel64
+#objdump: -dw -Mintel64
+#name: x86-64 sysenter (Intel64/Intel64)
+#source: x86-64-sysenter-amd.s
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+[ 	]*[a-f0-9]+:	0f 34                	sysenter *
+[ 	]*[a-f0-9]+:	0f 35                	sysexit *
+[ 	]*[a-f0-9]+:	0f 34                	sysenter *
+[ 	]*[a-f0-9]+:	0f 35                	sysexit *
+#pass
--- /dev/null
+++ b/gas/testsuite/gas/i386/x86-64-sysenter-mixed.d
@@ -0,0 +1,6 @@
+#as: -mintel64
+#objdump: -dw
+#name: x86-64 sysenter (Intel64/Default)
+#source: x86-64-sysenter-amd.s
+#dump: x86-64-sysenter-intel.d
+
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -107,6 +107,7 @@ static void OP_3DNowSuffix (int, int);
 static void CMP_Fixup (int, int);
 static void BadOp (void);
 static void REP_Fixup (int, int);
+static void SEP_Fixup (int, int);
 static void BND_Fixup (int, int);
 static void NOTRACK_Fixup (int, int);
 static void HLE_Fixup1 (int, int);
@@ -411,6 +412,7 @@ fetch_data (struct disassemble_info *inf
 #define EMCq { OP_EMC, q_mode }
 #define MXC { OP_MXC, 0 }
 #define OPSUF { OP_3DNowSuffix, 0 }
+#define SEP { SEP_Fixup, 0 }
 #define CMP { CMP_Fixup, 0 }
 #define XMM0 { XMM_Fixup, 0 }
 #define FXSAVE { FXSAVE_Fixup, 0 }
@@ -2711,8 +2713,8 @@ static const struct dis386 dis386_twobyt
   { "rdtsc",		{ XX }, 0 },
   { "rdmsr",		{ XX }, 0 },
   { "rdpmc",		{ XX }, 0 },
-  { "sysenter",		{ XX }, 0 },
-  { "sysexit",		{ XX }, 0 },
+  { "sysenter",		{ SEP }, 0 },
+  { "sysexit",		{ SEP }, 0 },
   { Bad_Opcode },
   { "getsec",		{ XX }, 0 },
   /* 38 */
@@ -11314,7 +11316,7 @@ static char scale_char;
 
 enum x86_64_isa
 {
-  amd64 = 0,
+  amd64 = 1,
   intel64
 };
 
@@ -14827,11 +14829,11 @@ OP_J (int bytemode, int sizeflag)
 	disp -= 0x100;
       break;
     case v_mode:
-      if (isa64 == amd64)
+      if (isa64 != intel64)
 	USED_REX (REX_W);
       if ((sizeflag & DFLAG)
 	  || (address_mode == mode_64bit
-	      && (isa64 != amd64 || (rex & REX_W))))
+	      && (isa64 == intel64 || (rex & REX_W))))
 	disp = get32s ();
       else
 	{
@@ -14848,7 +14850,7 @@ OP_J (int bytemode, int sizeflag)
 		       & ~((bfd_vma) 0xffff));
 	}
       if (address_mode != mode_64bit
-	  || (isa64 == amd64 && !(rex & REX_W)))
+	  || (isa64 != intel64 && !(rex & REX_W)))
 	used_prefixes |= (prefixes & PREFIX_DATA);
       break;
     default:
@@ -15598,6 +15600,18 @@ REP_Fixup (int bytemode, int sizeflag)
     }
 }
 
+static void
+SEP_Fixup (int bytemode ATTRIBUTE_UNUSED, int sizeflag ATTRIBUTE_UNUSED)
+{
+  if ( isa64 != amd64 )
+    return;
+
+  obufp = obuf;
+  BadOp ();
+  mnemonicendp = obufp;
+  ++codep;
+}
+
 /* For BND-prefixed instructions 0xF2 prefix should be displayed as
    "bnd".  */
 
--- a/opcodes/i386-opc.tbl
+++ b/opcodes/i386-opc.tbl
@@ -907,8 +907,10 @@ rdmsr, 0, 0xf32, None, 2, Cpu586, No_bSu
 cmpxchg8b, 1, 0xfc7, 0x1, 2, Cpu586, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf|IsLockable|NoRex64|HLEPrefixOk, { Qword|Unspecified|BaseIndex }
 
 // Pentium II/Pentium Pro extensions.
-sysenter, 0, 0xf34, None, 2, Cpu686, No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
-sysexit, 0, 0xf35, None, 2, Cpu686, No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysenter, 0, 0xf34, None, 2, Cpu686, Intel64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysenter, 0, 0xf34, None, 2, Cpu686|CpuNo64, AMD64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysexit, 0, 0xf35, None, 2, Cpu686, Intel64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysexit, 0, 0xf35, None, 2, Cpu686|CpuNo64, AMD64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
 fxsave, 1, 0xfae, 0x0, 2, CpuFXSR, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf, { Unspecified|BaseIndex }
 fxsave64, 1, 0xfae, 0x0, 2, CpuFXSR|Cpu64, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf|Rex64, { Unspecified|BaseIndex }
 fxrstor, 1, 0xfae, 0x1, 2, CpuFXSR, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf, { Unspecified|BaseIndex }