[PATCH 11/11] x86: adjust/correct VFRCZ{P,S}{S,D} decoding

[Jan Beulich]

The unnecessary XOP.L decoding had caught my eye, together with the not
really expected operand specifiers. Drop this decode step, and instead
make sure XOP.W and XOP.PP don't get ignored. For the latter, do this in
a form applicable to all XOP insns, rather than adding extra table
layers - there are no encodings with the field non-zero. Besides these
two, for the scalar forms XOP.L actually needs to also be zero.

opcodes/
2020-07-XX  Jan Beulich  <jbeulich@suse.com>

	* i386-dis.c (VEX_LEN_0FXOP_09_80, VEX_LEN_0FXOP_09_81): Delete.
	(VEX_LEN_0FXOP_09_82_W_0, VEX_LEN_0FXOP_09_83_W_0,
	VEX_W_0FXOP_09_80, VEX_W_0FXOP_09_81, VEX_W_0FXOP_09_82,
	VEX_W_0FXOP_09_83): New enumerators.
	(xop_table): Reference the above.
	(vex_len_table): Replace vfrczp* entries by vfrczs* ones.
	(vex_w_table): New VEX_W_0FXOP_09_80, VEX_W_0FXOP_09_81,
	VEX_W_0FXOP_09_82, and VEX_W_0FXOP_09_83 entries.
	(get_valid_dis386): Return bad_opcode for XOP.PP != 0.

---
I shall note that the state of XOP disassembly in general seems rather
poor, to a fair part because of custom code where generic code could be
used. I'll try to find time to clean this up.

--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -1814,8 +1814,8 @@ enum
   VEX_LEN_0FXOP_08_ED,
   VEX_LEN_0FXOP_08_EE,
   VEX_LEN_0FXOP_08_EF,
-  VEX_LEN_0FXOP_09_80,
-  VEX_LEN_0FXOP_09_81
+  VEX_LEN_0FXOP_09_82_W_0,
+  VEX_LEN_0FXOP_09_83_W_0,
 };
 
 enum
@@ -1956,6 +1956,11 @@ enum
   VEX_W_0F3ACE_P_2,
   VEX_W_0F3ACF_P_2,
 
+  VEX_W_0FXOP_09_80,
+  VEX_W_0FXOP_09_81,
+  VEX_W_0FXOP_09_82,
+  VEX_W_0FXOP_09_83,
+
   EVEX_W_0F10_P_1,
   EVEX_W_0F10_P_3,
   EVEX_W_0F11_P_1,
@@ -7862,10 +7867,10 @@ static const struct dis386 xop_table[][2
     { Bad_Opcode },
     { Bad_Opcode },
     /* 80 */
-    { VEX_LEN_TABLE (VEX_LEN_0FXOP_09_80) },
-    { VEX_LEN_TABLE (VEX_LEN_0FXOP_09_81) },
-    { "vfrczss", 	{ XM, EXd }, 0 },
-    { "vfrczsd", 	{ XM, EXq }, 0 },
+    { VEX_W_TABLE (VEX_W_0FXOP_09_80) },
+    { VEX_W_TABLE (VEX_W_0FXOP_09_81) },
+    { VEX_W_TABLE (VEX_W_0FXOP_09_82) },
+    { VEX_W_TABLE (VEX_W_0FXOP_09_83) },
     { Bad_Opcode },
     { Bad_Opcode },
     { Bad_Opcode },
@@ -9726,16 +9731,14 @@ static const struct dis386 vex_len_table
      { "vpcomuq",	{ XM, Vex128, EXx, VPCOM }, 0 },
   },
 
-  /* VEX_LEN_0FXOP_09_80 */
+  /* VEX_LEN_0FXOP_09_82_W_0 */
   {
-    { "vfrczps",	{ XM, EXxmm }, 0 },
-    { "vfrczps",	{ XM, EXymmq }, 0 },
+    { "vfrczss", 	{ XM, EXd }, 0 },
   },
 
-  /* VEX_LEN_0FXOP_09_81 */
+  /* VEX_LEN_0FXOP_09_83_W_0 */
   {
-    { "vfrczpd",	{ XM, EXxmm }, 0 },
-    { "vfrczpd",	{ XM, EXymmq }, 0 },
+    { "vfrczsd", 	{ XM, EXq }, 0 },
   },
 };
 
@@ -10063,6 +10066,22 @@ static const struct dis386 vex_w_table[]
     { Bad_Opcode },
     { "vgf2p8affineinvqb",  { XM, Vex, EXx, Ib }, 0 },
   },
+  /* VEX_W_0FXOP_09_80 */
+  {
+    { "vfrczps",	{ XM, EXx }, 0 },
+  },
+  /* VEX_W_0FXOP_09_81 */
+  {
+    { "vfrczpd",	{ XM, EXx }, 0 },
+  },
+  /* VEX_W_0FXOP_09_82 */
+  {
+    { VEX_LEN_TABLE (VEX_LEN_0FXOP_09_82_W_0) },
+  },
+  /* VEX_W_0FXOP_09_83 */
+  {
+    { VEX_LEN_TABLE (VEX_LEN_0FXOP_09_83_W_0) },
+  },
 
 #include "i386-dis-evex-w.h"
 };
@@ -11475,6 +11494,11 @@ get_valid_dis386 (const struct dis386 *d
       modrm.mod = (*codep >> 6) & 3;
       modrm.reg = (*codep >> 3) & 7;
       modrm.rm = *codep & 7;
+
+      /* No XOP encoding so far allows for a non-zero embedded prefix. Avoid
+	 having to decode the bits for every otherwise valid encoding.  */
+      if (vex.prefix)
+	return &bad_opcode;
       break;
 
     case USE_VEX_C4_TABLE: