From 2e4df506e0fd15f42810efe5eadce5b4f2d046d4 Mon Sep 17 00:00:00 2001
From: "H. Peter Anvin" <hpa@zytor.com>
Date: Mon, 29 Jul 2024 20:58:04 -0700
Subject: [PATCH] WIP: APX: can now generate code for tested APX instructions a
 Support generating code for APX instruction and add support for the {nf}
 prefix.

No disassembler support yet, and only a handful instructions encoded.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>
---
 Makefile.in       |   9 +-
 asm/assemble.c    | 456 ++++++++++++++++++++++++++++------------------
 asm/directiv.c    |   3 +-
 asm/parser.c      |  59 +++---
 asm/tokens.dat    |   4 +
 include/nasm.h    |  51 ++++--
 include/nasmlib.h |  10 +-
 include/opflags.h |  15 +-
 test/apx.asm      |  33 ++++
 x86/bytecode.txt  |  25 +--
 x86/disp8.c       |   6 +-
 x86/insns.dat     |  73 ++++----
 x86/insns.pl      | 106 ++++++++---
 13 files changed, 543 insertions(+), 307 deletions(-)
 create mode 100644 test/apx.asm

diff --git a/Makefile.in b/Makefile.in
index 997681530..223abe38d 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -113,8 +113,8 @@ MANIFEST = @MANIFEST@
 NASM    = asm/nasm.$(O)
 NDISASM = disasm/ndisasm.$(O)
 
-PROGOBJ = $(NASM) $(NDISASM)
-PROGS   = nasm$(X) ndisasm$(X)
+PROGOBJ = $(NASM) # $(NDISASM)
+PROGS   = nasm$(X) # ndisasm$(X)
 
 LIBOBJ_NW = stdlib/snprintf.$(O) stdlib/vsnprintf.$(O) stdlib/strlcpy.$(O) \
 	stdlib/strnlen.$(O) stdlib/strrchrnul.$(O) \
@@ -160,8 +160,9 @@ LIBOBJ_NW = stdlib/snprintf.$(O) stdlib/vsnprintf.$(O) stdlib/strlcpy.$(O) \
 	output/outobj.$(O) output/outas86.$(O) \
 	output/outdbg.$(O) output/outieee.$(O) output/outmacho.$(O) \
 	output/codeview.$(O) \
-	\
-	disasm/disasm.$(O) disasm/sync.$(O)
+
+#	\
+#	disasm/disasm.$(O) disasm/sync.$(O)
 
 # Warnings depend on all source files, so handle them separately
 WARNOBJ   = asm/warnings.$(O)
diff --git a/asm/assemble.c b/asm/assemble.c
index 41a3d45b8..e1ff711f3 100644
--- a/asm/assemble.c
+++ b/asm/assemble.c
@@ -102,9 +102,9 @@ static enum match_result find_match(const struct itemplate **tempp,
 static enum match_result matches(const struct itemplate *, insn *, int bits);
 static opflags_t regflag(const operand *);
 static int32_t regval(const operand *);
-static int rexflags(int, opflags_t, int);
-static int op_rexflags(const operand *, int);
-static int op_evexflags(const operand *, int, uint8_t);
+static uint32_t rexflags(int, opflags_t, uint32_t);
+static uint32_t op_rexflags(const operand *, uint32_t);
+static uint32_t op_evexflags(const operand *, uint32_t);
 static void add_asp(insn *, int);
 
 static int process_ea(operand *, ea *, int, int, opflags_t,
@@ -151,31 +151,37 @@ static const char *size_name(int size)
     }
 }
 
-static void warn_overflow(int size)
+static void warn_overflow(int size, const char *prefix)
 {
-    nasm_warn(ERR_PASS2 | WARN_NUMBER_OVERFLOW, "%s data exceeds bounds",
-               size_name(size));
+    nasm_warn(ERR_PASS2 | WARN_NUMBER_OVERFLOW,
+              "%s%s data exceeds bounds",
+              prefix, size_name(size));
 }
 
 static void warn_overflow_const(int64_t data, int size)
 {
     if (overflow_general(data, size))
-        warn_overflow(size);
+        warn_overflow(size, "");
 }
 
 static void warn_overflow_out(int64_t data, int size, enum out_flags flags)
 {
     bool err;
+    const char *prefix;
 
-    if (flags & OUT_SIGNED)
+    if (flags & OUT_SIGNED) {
+        prefix = "signed ";
         err = overflow_signed(data, size);
-    else if (flags & OUT_UNSIGNED)
+    } else if (flags & OUT_UNSIGNED) {
+        prefix = "unsigned ";
         err = overflow_unsigned(data, size);
-    else
+    } else {
+        prefix = "";
         err = overflow_general(data, size);
+    }
 
     if (err)
-        warn_overflow(size);
+        warn_overflow(size, prefix);
 }
 
 /*
@@ -462,6 +468,24 @@ static void out_rawbyte(struct out_data *data, uint8_t byte)
     out(data);
 }
 
+static void out_rawword(struct out_data *data, uint16_t value)
+{
+    uint16_t buf = cpu_to_le16(value);
+    data->type = OUT_RAWDATA;
+    data->data = &buf;
+    data->size = 2;
+    out(data);
+}
+
+static void out_rawdword(struct out_data *data, uint32_t value)
+{
+    uint32_t buf = cpu_to_le32(value);
+    data->type = OUT_RAWDATA;
+    data->data = &buf;
+    data->size = 4;
+    out(data);
+}
+
 static inline void out_reserve(struct out_data *data, uint64_t size)
 {
     data->type = OUT_RESERVE;
@@ -1213,7 +1237,6 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
     const uint8_t *codes = temp->code;
     int64_t length = 0;
     uint8_t c;
-    int rex_mask = ~0;
     int op1, op2;
     struct operand *opx;
     uint8_t opex = 0;
@@ -1223,12 +1246,14 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
     enum reg_enum mib_index = R_none;   /* For a separate index reg form */
     const char *errmsg;
 
-    ins->rex = 0;               /* Ensure REX is reset */
+    ins->rex     = 0;           /* Ensure REX is reset */
+    ins->evex    = 0;		/* Ensure EVEX is reset */
+    ins->vexreg  = 0;           /* No V register */
+    ins->vex_cm  = 0;           /* No implicit map */
     eat = EA_SCALAR;            /* Expect a scalar EA */
-    memset(ins->evex_p, 0, 3);  /* Ensure EVEX is reset */
 
-    if (ins->prefixes[PPS_OSIZE] == P_O64)
-        ins->rex |= REX_W;
+    /* Default operand size */
+    ins->op_size = bits == 16 ? 32 : 16;
 
     (void)segment;              /* Don't warn that this parameter is unused */
     (void)offset;               /* Don't warn that this parameter is unused */
@@ -1250,8 +1275,7 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             break;
 
         case4(010):
-            ins->rex |=
-                op_rexflags(opx, REX_B|REX_H|REX_P|REX_W);
+            ins->rex |= op_rexflags(opx, REX_rB);
             codes++, length++;
             break;
 
@@ -1315,7 +1339,7 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             c = *codes++;
             op2 = (op2 & ~3) | ((c >> 3) & 3);
             opx = &ins->oprs[op2];
-            ins->rex |= op_rexflags(opx, REX_R|REX_H|REX_P|REX_W);
+            ins->rex |= op_rexflags(opx, REX_rR);
             length++;
             break;
 
@@ -1330,19 +1354,19 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             break;
 
         case4(0240):
-            ins->rex |= REX_EV;
             ins->vexreg = regval(opx);
-            ins->evex_p[2] |= op_evexflags(opx, EVEX_P2VP, 2); /* High-16 NDS */
-            ins->vex_cm = *codes++;
-            ins->vex_wlp = *codes++;
-            ins->evex_tuple = (*codes++ - 0300);
-            break;
+            goto evex_common;
 
         case 0250:
-            ins->rex |= REX_EV;
             ins->vexreg = 0;
-            ins->vex_cm = *codes++;
-            ins->vex_wlp = *codes++;
+            goto evex_common;
+
+        evex_common:
+            ins->rex |= REX_EV;
+            ins->evex = 0x62;
+            ins->evex += *codes++ << 8;
+            ins->evex += *codes++ << 16;
+            ins->evex += *codes++ << 24;
             ins->evex_tuple = (*codes++ - 0300);
             break;
 
@@ -1351,16 +1375,16 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             break;
 
         case4(0260):
-            ins->rex |= REX_V;
             ins->vexreg = regval(opx);
-            ins->vex_cm = *codes++;
-            ins->vex_wlp = *codes++;
-            break;
+            goto vex_common;
 
         case 0270:
-            ins->rex |= REX_V;
             ins->vexreg = 0;
-            ins->vex_cm = *codes++;
+            goto vex_common;
+
+        vex_common:
+            ins->rex |= REX_V;
+            ins->vex_cm  = *codes++;
             ins->vex_wlp = *codes++;
             break;
 
@@ -1405,39 +1429,54 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
              *!   The operand prefix will be ignored by the assembler.
              */
             enum prefixes pfx = ins->prefixes[PPS_OSIZE];
-            if (pfx == P_O16)
-                break;
-            if (pfx != P_none)
+            ins->op_size = 16;
+            if (bits != 16 && pfx == P_OSP) {
+                /* Allow osp prefix as is */
+            } else if (pfx != P_none && pfx != P_O16) {
                 nasm_warn(WARN_PREFIX_OPSIZE|ERR_PASS2,
                           "invalid operand size prefix, must be o16");
-            else
+            } else {
                 ins->prefixes[PPS_OSIZE] = P_O16;
+            }
             break;
         }
 
         case 0321:
         {
             enum prefixes pfx = ins->prefixes[PPS_OSIZE];
-            if (pfx == P_O32)
-                break;
-            if (pfx != P_none)
+            ins->op_size = 32;
+            if (bits == 16 && pfx == P_OSP) {
+                /* Allow osp prefix as is */
+            } else if (pfx != P_none && pfx != P_O32) {
                 nasm_warn(WARN_PREFIX_OPSIZE|ERR_PASS2,
                           "invalid operand size prefix, must be o32");
-            else
+            } else {
                 ins->prefixes[PPS_OSIZE] = P_O32;
+            }
             break;
         }
 
         case 0322:
             break;
 
-        case 0323:
-            rex_mask &= ~REX_W;
-            break;
-
         case 0324:
             ins->rex |= REX_W;
+            /* fall through */
+
+        case 0323:
+        {
+            enum prefixes pfx = ins->prefixes[PPS_OSIZE];
+            ins->op_size = 64;
+            if (pfx == P_OSP) {
+                /* Ignore operand size prefix */
+            } else if (pfx != P_none && pfx != P_O64) {
+                nasm_warn(WARN_PREFIX_OPSIZE|ERR_PASS2,
+                          "invalid operand size prefix, must be o64");
+            } else {
+                ins->prefixes[PPS_OSIZE] = P_none;
+            }
             break;
+        }
 
         case 0325:
             ins->rex |= REX_NH;
@@ -1498,6 +1537,19 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
                 ins->prefixes[PPS_WAIT] = P_WAIT;
             break;
 
+        case4(0350):
+        case4(0354):
+            ins->rex |= REX_2 | ((c & 4) << (13-2)); /* X1 bit */
+            goto rexx_common;
+
+        case4(0344):
+            goto rexx_common;
+
+        rexx_common:
+            ins->rex |= (c & 2) << (3-1); /* W bit */
+            ins->vex_cm = c & 1;
+            break;
+
         case 0360:
             break;
 
@@ -1570,20 +1622,15 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
 
                 if (op_er_sae && (op_er_sae->decoflags & (ER | SAE))) {
                     /* set EVEX.b */
-                    ins->evex_p[2] |= EVEX_P2B;
+                    ins->evex ^= EVEX_P2B;
                     if (op_er_sae->decoflags & ER) {
                         /* set EVEX.RC (rounding control) */
-                        ins->evex_p[2] |= ((ins->evex_rm - BRC_RN) << 5)
-                                          & EVEX_P2RC;
-                    }
-                } else {
-                    /* set EVEX.L'L (vector length) */
-                    ins->evex_p[2] |= ((ins->vex_wlp << (5 - 2)) & EVEX_P2LL);
-                    ins->evex_p[1] |= ((ins->vex_wlp << (7 - 4)) & EVEX_P1W);
-                    if (opy->decoflags & BRDCAST_MASK) {
-                        /* set EVEX.b */
-                        ins->evex_p[2] |= EVEX_P2B;
+                        ins->evex ^= ((ins->evex_rm - BRC_RN) << 29)
+                            & EVEX_P2RC;
                     }
+                } else if (opy->decoflags & BRDCAST_MASK) {
+                    /* set EVEX.b */
+                    ins->evex ^= EVEX_P2B;
                 }
 
                 if (itemp_has(temp, IF_MIB)) {
@@ -1622,8 +1669,6 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
         }
     }
 
-    ins->rex &= rex_mask;
-
     if (ins->rex & REX_NH) {
         if (ins->rex & REX_H) {
             nasm_nonfatal("instruction cannot use high registers");
@@ -1644,70 +1689,93 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             return -1;
         break;
     case P_REX:
-        if (ins->rex & (REX_V|REX_EV))
+        if (ins->rex & (REX_V|REX_EV|REX_2))
             return -1;
         ins->rex |= REX_P;      /* Force REX prefix */
         break;
+    case P_REX2:
+        if (ins->rex & (REX_V|REX_EV))
+            return -1;
+        ins->rex |= REX_P | REX_2; /* Force REX2 prefix */
+        break;
     default:
         break;
     }
 
     if (ins->rex & (REX_V | REX_EV)) {
-        int bad32 = REX_R|REX_W|REX_X|REX_B;
+        uint32_t bad32 = REX_BXR;
 
         if (ins->rex & REX_H) {
-            nasm_nonfatal("cannot use high register in AVX instruction");
+            nasm_nonfatal("cannot use high byte register in this instruction");
             return -1;
         }
-        switch (ins->vex_wlp & 060) {
-        case 000:
-        case 040:
-            ins->rex &= ~REX_W;
-            break;
-        case 020:
-            ins->rex |= REX_W;
-            bad32 &= ~REX_W;
-            break;
-        case 060:
-            /* Follow REX_W */
-            break;
+        if (itemp_has(temp, IF_WW)) {
+            bad32 |= REX_W;
+        } else {
+            ins->rex = (ins->rex & ~REX_W) | ((ins->vex_wlp >> (7-3)) & REX_W);
         }
 
         if (bits != 64 && ((ins->rex & bad32) || ins->vexreg > 7)) {
             nasm_nonfatal("invalid operands in non-64-bit mode");
             return -1;
-        } else if (!(ins->rex & REX_EV) &&
-                   ((ins->vexreg > 15) || (ins->evex_p[0] & 0xf0))) {
-            nasm_nonfatal("invalid high-16 register in non-AVX-512");
+        }
+
+        if (ins->rex & REX_EV) {
+            /* EVEX */
+            length += 4;
+        } else {
+            /* VEX */
+            if (ins->vexreg > 15 || (ins->rex & REX_BXR1))
+                nasm_nonfatal("invalid high-16 register in non-AVX-512");
+            return -1;
+
+            if (ins->vex_cm != 1 || (ins->rex & REX_BXR0) ||
+                ins->prefixes[PPS_REX] == P_VEX3) {
+                /* VEX3 required */
+                if (ins->prefixes[PPS_REX] == P_VEX2)
+                    nasm_nonfatal("instruction not encodable with {vex2} prefix");
+                length += 3;
+            } else {
+                /* VEX2 available */
+                length += 2;
+            }
+        }
+    } else if (ins->rex & (REX_BXR1 | REX_2)) {
+        /* REX2 prefix needed */
+        if (ins->rex & REX_H) {
+            nasm_nonfatal("cannot use high byte register in rex2 instruction");
             return -1;
         }
-        if (ins->rex & REX_EV) {
-            length += 4;
-        } else if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)) ||
-                 ins->prefixes[PPS_REX] == P_VEX3) {
-            if (ins->prefixes[PPS_REX] == P_VEX2)
-                nasm_nonfatal("instruction not encodable with {vex2} prefix");
-            length += 3;
-        } else {
-            length += 2;
+        if (bits != 64) {
+            nasm_nonfatal("invalid operands in non-64-bit mode");
+            return -1;
         }
+        if (!itemp_has(temp, IF_REX2) || !iflag_test(&cpu, IF_APX)) {
+            nasm_nonfatal("invalid operands in non-APX mode");
+            return -1;
+        }
+
+        ins->rex |= REX_2 | REX_P;
+        length += 2;
     } else if (ins->rex & REX_MASK) {
         if (ins->rex & REX_H) {
             nasm_nonfatal("cannot use high byte register in rex instruction");
             return -1;
         } else if (bits == 64) {
-            length++;
-        } else if ((ins->rex & REX_L) &&
-                   !(ins->rex & (REX_P|REX_W|REX_X|REX_B)) &&
+            ins->rex |= REX_P;
+        } else if ((ins->rex & (REX_L|REX_W|REX_BXR)) == (REX_L|REX_R) &&
                    iflag_cpu_level_ok(&cpu, IF_X86_64)) {
             /* LOCK-as-REX.R */
             assert_no_prefix(ins, PPS_LOCK);
             lockcheck = false;  /* Already errored, no need for warning */
-            length++;
+            ins->rex &= ~REX_P;
         } else {
             nasm_nonfatal("invalid operands in non-64-bit mode");
             return -1;
         }
+
+        /* Implicitly encoded legacy prefixes */
+        length += 1 + (ins->vex_cm > 0) + (ins->vex_cm > 1);
     }
 
     if (lockcheck && has_prefix(ins, PPS_LOCK, P_LOCK)) {
@@ -1754,13 +1822,30 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
 
 static inline void emit_rex(struct out_data *data, insn *ins)
 {
-    if (data->bits == 64) {
-        if ((ins->rex & REX_MASK) &&
-            !(ins->rex & (REX_V | REX_EV)) &&
-            !ins->rex_done) {
-            uint8_t rex = (ins->rex & REX_MASK) | REX_P;
+    if (data->bits != 64 || ins->rex_done)
+        return;
+
+    ins->rex_done = true;
+
+    if (ins->rex & (REX_V | REX_EV))
+        return;                 /* Handled elsewhere */
+
+    if (ins->rex & REX_2) {
+        uint16_t rex2 = 0x00d5;
+        rex2 |= (ins->rex & (REX_BXR0|REX_W)) << 8;
+        rex2 |= (ins->rex & REX_BXR1);
+        rex2 |= (ins->vex_cm & 1) << 15;
+        out_rawword(data, rex2);
+    } else {
+        uint8_t rex = ins->rex & REX_MASK;
+        if (rex & REX_P)
             out_rawbyte(data, rex);
-            ins->rex_done = true;
+        if (ins->vex_cm) {
+            out_rawbyte(data, 0x0f);
+            if (ins->vex_cm > 1) {
+                /* Map 2 = 0F 38, map 3 = 0F 3A */
+                out_rawbyte(data, 0x36 + (ins->vex_cm << 1));
+            }
         }
     }
 }
@@ -1853,25 +1938,30 @@ static int emit_prefix(struct out_data *data, const int bits, insn *ins)
             c = 0x67;
             break;
         case P_O16:
-            if (bits != 16)
+            if (bits != 16 && !(ins->rex & (REX_W|REX_V|REX_EV))) {
                 c = 0x66;
+            }
             break;
         case P_O32:
-            if (bits == 16)
+            if (bits == 16 && !(ins->rex & (REX_W|REX_V|REX_EV))) {
                 c = 0x66;
+            }
             break;
         case P_O64:
-            /* REX.W */
+            /* Handled via REX.W */
             break;
         case P_OSP:
-            c = 0x66;
+            if (!(ins->rex & (REX_2|REX_V|REX_EV)))
+                c = 0x66;
             break;
         case P_REX:
         case P_VEX:
         case P_EVEX:
         case P_VEX3:
         case P_VEX2:
+        case P_REX2:
         case P_NOBND:
+        case P_NF:
         case P_none:
             break;
         default:
@@ -2054,35 +2144,36 @@ static void gencode(struct out_data *data, insn *ins)
         case4(0240):
         case 0250:
             codes += 3;
-            ins->evex_p[2] |= op_evexflags(&ins->oprs[0],
-                                           EVEX_P2Z | EVEX_P2AAA, 2);
-            ins->evex_p[2] ^= EVEX_P2VP;        /* 1's complement */
-            bytes[0] = 0x62;
-            /* EVEX.X can be set by either REX or EVEX for different reasons */
-            bytes[1] = ((((ins->rex & 7) << 5) |
-                         (ins->evex_p[0] & (EVEX_P0X | EVEX_P0RP))) ^ 0xf0) |
-                       (ins->vex_cm & EVEX_P0MM);
-            bytes[2] = ((ins->rex & REX_W) << (7 - 3)) |
-                       ((~ins->vexreg & 15) << 3) |
-                       (1 << 2) | (ins->vex_wlp & 3);
-            bytes[3] = ins->evex_p[2];
-            out_rawdata(data, bytes, 4);
+            ins->evex ^= op_evexflags(&ins->oprs[0], EVEX_P2Z|EVEX_P2AAA);
+            ins->evex ^= (ins->rex & REX_BXR0) << 12; /* R0 B0 X0 */
+            ins->evex ^= (ins->rex & REX_B1)   >> (12-11);
+            ins->evex ^= (ins->rex & REX_X1)   << (18-13);
+            ins->evex ^= (ins->rex & REX_R1)   << (15-14);
+            ins->evex ^= (ins->rex & REX_W)    << (23-3);
+            ins->evex ^= (ins->vexreg & 15) << 19;
+            ins->evex ^= (ins->vexreg & 16) << (27 - 4);
+            if (ins->prefixes[PPS_NF] == P_NF)
+                ins->evex ^= EVEX_P2NF;
+            out_rawdword(data, ins->evex);
             break;
 
         case4(0260):
         case 0270:
             codes += 2;
-            if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)) ||
+            if (ins->vex_cm != 1 ||
+                (ins->rex & (REX_W|REX_X|REX_B)) ||
                 ins->prefixes[PPS_REX] == P_VEX3) {
                 bytes[0] = (ins->vex_cm >> 6) ? 0x8f : 0xc4;
-                bytes[1] = (ins->vex_cm & 31) | ((~ins->rex & 7) << 5);
+                bytes[1] = (ins->vex_cm & 31) |
+                    ((~ins->rex & REX_BXR0) << 5);
                 bytes[2] = ((ins->rex & REX_W) << (7-3)) |
-                    ((~ins->vexreg & 15)<< 3) | (ins->vex_wlp & 07);
+                    ((~ins->vexreg & 15) << 3);
                 out_rawdata(data, bytes, 3);
             } else {
                 bytes[0] = 0xc5;
                 bytes[1] = ((~ins->rex & REX_R) << (7-2)) |
-                    ((~ins->vexreg & 15) << 3) | (ins->vex_wlp & 07);
+                    ((~ins->vexreg & 15) << 3) |
+                    (ins->vex_wlp & 07);
                 out_rawdata(data, bytes, 2);
             }
             break;
@@ -2098,14 +2189,7 @@ static void gencode(struct out_data *data, insn *ins)
             int s;
 
             if (absolute_op(opx)) {
-                if (ins->rex & REX_W)
-                    s = 64;
-                else if (ins->prefixes[PPS_OSIZE] == P_O16)
-                    s = 16;
-                else if (ins->prefixes[PPS_OSIZE] == P_O32)
-                    s = 32;
-                else
-                    s = bits;
+                s = ins->op_size;
 
                 um = (uint64_t)2 << (s-1);
                 uv = opx->offset;
@@ -2159,10 +2243,7 @@ static void gencode(struct out_data *data, insn *ins)
 
         case 0322:
         case 0323:
-            break;
-
         case 0324:
-            ins->rex |= REX_W;
             break;
 
         case 0325:
@@ -2202,6 +2283,11 @@ static void gencode(struct out_data *data, insn *ins)
         case 0341:
             break;
 
+        case4(0344):
+        case4(0350):
+        case4(0354):
+            break;
+
         case 0360:
             break;
 
@@ -2293,7 +2379,7 @@ static void gencode(struct out_data *data, insn *ins)
                         if (overflow_general(opy->offset, asize) ||
                             sext(opy->offset, ins->addr_size) !=
                             sext(opy->offset, ea_data.bytes << 3))
-                            warn_overflow(ea_data.bytes);
+                            warn_overflow(ea_data.bytes, "displacement ");
 
                         out_imm(data, opy, ea_data.bytes,
                                 (asize > ea_data.bytes)
@@ -2320,12 +2406,18 @@ static opflags_t regflag(const operand * o)
 
 static int32_t regval(const operand * o)
 {
+    /*
+     * Certain instruction patterns allow an immediate to be put
+     * into a register field
+     */
+    if (o->type & IMMEDIATE)
+        return o->offset;
     if (!is_register(o->basereg))
         nasm_panic("invalid operand passed to regval()");
     return nasm_regvals[o->basereg];
 }
 
-static int op_rexflags(const operand * o, int mask)
+static uint32_t op_rexflags(const operand * o, uint32_t mask)
 {
     opflags_t flags;
     int val;
@@ -2339,12 +2431,16 @@ static int op_rexflags(const operand * o, int mask)
     return rexflags(val, flags, mask);
 }
 
-static int rexflags(int val, opflags_t flags, int mask)
+static uint32_t rexflags(int val, opflags_t flags, uint32_t mask)
 {
-    int rex = 0;
+    uint32_t rex = 0;
 
-    if (val >= 0 && (val & 8))
-        rex |= REX_B|REX_X|REX_R;
+    if (val >= 0) {
+        if (val & 8)
+            rex |= REX_B|REX_X|REX_R;
+        if (val & 16)
+            rex |= REX_B1|REX_X1|REX_R1;
+    }
     if (flags & BITS64)
         rex |= REX_W;
     if (!(REG_HIGH & ~flags))                   /* AH, CH, DH, BH */
@@ -2355,35 +2451,21 @@ static int rexflags(int val, opflags_t flags, int mask)
     return rex & mask;
 }
 
-static int evexflags(int val, decoflags_t deco,
-                     int mask, uint8_t byte)
+static uint32_t evexflags(decoflags_t deco, uint32_t mask)
 {
-    int evex = 0;
+    uint32_t evex = 0;
+
+    if (deco & Z)
+        evex |= EVEX_P2Z;
+    if (deco & OPMASK_MASK)
+        evex |= (deco << 24) & EVEX_P2AAA;
 
-    switch (byte) {
-    case 0:
-        if (val >= 0 && (val & 16))
-            evex |= (EVEX_P0RP | EVEX_P0X);
-        break;
-    case 2:
-        if (val >= 0 && (val & 16))
-            evex |= EVEX_P2VP;
-        if (deco & Z)
-            evex |= EVEX_P2Z;
-        if (deco & OPMASK_MASK)
-            evex |= deco & EVEX_P2AAA;
-        break;
-    }
     return evex & mask;
 }
 
-static int op_evexflags(const operand * o, int mask, uint8_t byte)
+static uint32_t op_evexflags(const operand * o, uint32_t mask)
 {
-    int val;
-
-    val = nasm_regvals[o->basereg];
-
-    return evexflags(val, o->decoflags, mask, byte);
+    return evexflags(o->decoflags, mask);
 }
 
 static enum match_result find_match(const struct itemplate **tempp,
@@ -2501,7 +2583,7 @@ static enum match_result matches(const struct itemplate *itemp,
 {
     opflags_t size[MAX_OPERANDS], asize;
     bool opsizemissing = false;
-    int i, oprs;
+    int i, oprs, matchop;
 
     /*
      * Check the opcode
@@ -2540,6 +2622,10 @@ static enum match_result matches(const struct itemplate *itemp,
             bits != 64)
             return MERR_ENCMISMATCH;
         break;
+    case P_REX2:
+        if (!itemp_has(itemp, IF_REX2) || bits != 64)
+            return MERR_ENCMISMATCH;
+        break;
     default:
         if (itemp_has(itemp, IF_EVEX)) {
             if (!iflag_test(&cpu, IF_EVEX))
@@ -2727,18 +2813,25 @@ static enum match_result matches(const struct itemplate *itemp,
     /*
      * Check operand sizes
      */
-    if (itemp_has(itemp, IF_SM) || itemp_has(itemp, IF_SM2)) {
-        oprs = (itemp_has(itemp, IF_SM2) ? 2 : itemp->operands);
-        for (i = 0; i < oprs; i++) {
-            asize = itemp->opd[i] & SIZE_MASK;
-            if (asize) {
-                for (i = 0; i < oprs; i++)
-                    size[i] = asize;
-                break;
-            }
-        }
-    } else {
+    matchop = oprs = 0;
+    if (itemp_has(itemp, IF_SM)) {
         oprs = itemp->operands;
+        matchop = 0;
+    } else if (itemp_has(itemp, IF_SM2)) {
+        oprs = 2;
+        matchop = 0;
+    } else if (itemp_has(itemp, IF_SM23)) {
+        oprs = 3;
+        matchop = 1;
+    }
+
+    for (i = matchop; i < oprs; i++) {
+        asize = itemp->opd[i] & SIZE_MASK;
+        if (asize) {
+            for (i = matchop; i < oprs; i++)
+                size[i] = asize;
+            break;
+        }
     }
 
     for (i = 0; i < itemp->operands; i++) {
@@ -2759,6 +2852,13 @@ static enum match_result matches(const struct itemplate *itemp,
     if (itemp_has(itemp, (bits == 64 ? IF_NOLONG : IF_LONG)))
         return MERR_BADMODE;
 
+    /*
+     * {nf} prefix used? Must be permitted.
+     */
+    if (has_prefix(instruction, PPS_NF, P_NF) &&
+        !itemp_has(itemp, IF_NF))
+        return MERR_ENCMISMATCH;
+
     /*
      * If we have a HLE prefix, look for the NOHLE flag
      */
@@ -2817,9 +2917,7 @@ static int process_ea(operand *input, ea *output, int bits,
     output->disp8   = 0;
 
     /* REX flags for the rfield operand */
-    output->rex     |= rexflags(rfield, rflags, REX_R | REX_P | REX_W | REX_H);
-    /* EVEX.R' flag for the REG operand */
-    ins->evex_p[0]  |= evexflags(rfield, 0, EVEX_P0RP, 0);
+    output->rex     |= rexflags(rfield, rflags, REX_rR);
 
     if (is_class(REGISTER, input->type)) {
         /*
@@ -2837,8 +2935,7 @@ static int process_ea(operand *input, ea *output, int bits,
             goto err;
         }
 
-        output->rex         |= op_rexflags(input, REX_B | REX_P | REX_W | REX_H);
-        ins->evex_p[0]      |= op_evexflags(input, EVEX_P0X, 0);
+        output->rex         |= op_rexflags(input, REX_rB);
         output->sib_present = false;    /* no SIB necessary */
         output->bytes       = 0;        /* no offset necessary either */
         output->modrm       = GEN_MODRM(3, rfield, nasm_regvals[input->basereg]);
@@ -2988,9 +3085,8 @@ static int process_ea(operand *input, ea *output, int bits,
                                 : ((ix & YMMREG & ~REG_EA)
                                 ? EA_YMMVSIB : EA_XMMVSIB));
 
-                output->rex    |= rexflags(it, ix, REX_X);
-                output->rex    |= rexflags(bt, bx, REX_B);
-                ins->evex_p[2] |= evexflags(it, 0, EVEX_P2VP, 2);
+                output->rex    |= rexflags(it, ix, REX_rX);
+                output->rex    |= rexflags(bt, bx, REX_rB);
 
                 index = it & 7; /* it is known to be != -1 */
 
@@ -3117,8 +3213,8 @@ static int process_ea(operand *input, ea *output, int bits,
                     (s != 1 && s != 2 && s != 4 && s != 8 && it != -1))
                     goto err;        /* wrong, for various reasons */
 
-                output->rex |= rexflags(it, ix, REX_X);
-                output->rex |= rexflags(bt, bx, REX_B);
+                output->rex |= rexflags(it, ix, REX_rX);
+                output->rex |= rexflags(bt, bx, REX_rB);
 
                 if (it == -1 && (bt & 7) != REG_NUM_ESP && !(eaflags & EAF_SIB)) {
                     /* no SIB needed */
@@ -3377,7 +3473,7 @@ static void add_asp(insn *ins, int addrbits)
         ins->addr_size = addrbits;
     } else if (valid & ((addrbits == 32) ? 16 : 32)) {
         /* Add an address size prefix */
-        ins->prefixes[PPS_ASIZE] = (addrbits == 32) ? P_A16 : P_A32;;
+        ins->prefixes[PPS_ASIZE] = (addrbits == 32) ? P_A16 : P_A32;
         ins->addr_size = (addrbits == 32) ? 16 : 32;
     } else {
         /* Impossible... */
diff --git a/asm/directiv.c b/asm/directiv.c
index 7260580f7..24b10c576 100644
--- a/asm/directiv.c
+++ b/asm/directiv.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 1996-2022 The NASM Authors - All Rights Reserved
+ *   Copyright 1996-2024 The NASM Authors - All Rights Reserved
  *   See the file AUTHORS included with the NASM distribution for
  *   the specific copyright holders.
  *
@@ -111,6 +111,7 @@ void set_cpu(const char *value)
         { "any", IF_ANY },
         { "all", IF_ANY },
         { "latevex", IF_LATEVEX },
+        { "apx", IF_APX },
         { "evex", IF_EVEX },
         { "vex", IF_VEX },
         { NULL, 0 }
diff --git a/asm/parser.c b/asm/parser.c
index aa2eb076c..8e8bf0b98 100644
--- a/asm/parser.c
+++ b/asm/parser.c
@@ -655,7 +655,9 @@ static opflags_t imm_flags(int64_t n, opflags_t flags)
 {
     if (n == 1)
         flags |= UNITY;
-    if ((uint64_t)n <= 15)
+
+    /* Allow FOURBITS matching for negative values, so things like ~0 work */
+    if (n >= -16 && n <= 15)
         flags |= FOURBITS;
 
     if (optimizing.level < 0 || (flags & STRICT))
@@ -1048,26 +1050,36 @@ restart_parse:
             goto mref_more;
         }
 
-        if (i == ':' && (mref || !far_jmp_ok)) {
-            /* segment override? */
-            mref = true;
+        if (i == ':') {
+            bool ok_reg = is_register(value->type) &&
+                value->value == 1 && !value[1].type;
 
-            /*
-             * Process the segment override.
-             */
-            if (!IS_SREG(value->type) || value->value != 1 ||
-                value[1].type != 0) {
-                nasm_nonfatal("invalid segment override");
-            } else if (result->prefixes[PPS_SEG]) {
-                nasm_nonfatal("instruction has conflicting segment overrides");
-            } else {
-                result->prefixes[PPS_SEG] = value->type;
-                if (IS_FSGS(value->type))
-                    op->eaflags |= EAF_FSGS;
+            if (!mref && ok_reg && !IS_SREG(value->type)) {
+                /*
+                 * Register pair syntax; this terminates the expression
+                 * as if it had ended in a comma, but sets the COLON flag
+                 * on the operand further down.
+                 */
+            } else if (mref || !far_jmp_ok) {
+                /* segment override? */
+                mref = true;
+
+                /*
+                 * Process the segment override.
+                 */
+                if (!ok_reg || !IS_SREG(value->type)) {
+                    nasm_nonfatal("invalid segment override");
+                } else if (result->prefixes[PPS_SEG]) {
+                    nasm_nonfatal("instruction has conflicting segment overrides");
+                } else {
+                    result->prefixes[PPS_SEG] = value->type;
+                    if (IS_FSGS(value->type))
+                        op->eaflags |= EAF_FSGS;
+                }
+
+                i = stdscan(NULL, &tokval); /* then skip the colon */
+                goto mref_more;
             }
-
-            i = stdscan(NULL, &tokval); /* then skip the colon */
-            goto mref_more;
         }
 
         mib = false;
@@ -1264,8 +1276,11 @@ restart_parse:
                     regset_size = 0;
                 }
 
-                /* clear overrides, except TO which applies to FPU regs */
-                if (op->type & ~TO) {
+                /*
+                 * Clear overrides, except TO which applies to FPU regs
+                 * and colon which is used in register pair syntax
+                 */
+                if (op->type & ~(TO | COLON)) {
                     /*
                      * we want to produce a warning iff the specified size
                      * is different from the register size
@@ -1292,7 +1307,7 @@ restart_parse:
                         goto fail;
                 }
 
-                op->type      &= TO;
+                op->type      &= TO | COLON;
                 op->type      |= REGISTER;
                 op->type      |= nasm_reg_flags[value->type];
                 op->type      |= (regset_size >> 1) << REGSET_SHIFT;
diff --git a/asm/tokens.dat b/asm/tokens.dat
index 6e7dc67f9..5205c48dc 100644
--- a/asm/tokens.dat
+++ b/asm/tokens.dat
@@ -77,11 +77,15 @@ wait
 
 % TOKEN_PREFIX, PPS_REX, TFLAG_BRC, P_*
 rex
+rex2
 evex
 vex
 vex3
 vex2
 
+% TOKEN_PREFIX, PPS_NF, TFLAG_BRC, P_*
+nf
+
 % TOKEN_SIZE, SIZE_*, 0, S_*
 byte
 word
diff --git a/include/nasm.h b/include/nasm.h
index 829f673cd..ee7fb9bab 100644
--- a/include/nasm.h
+++ b/include/nasm.h
@@ -535,12 +535,6 @@ enum {
  * -----------------------------------------------------------
  */
 
-/* Verify value to be a valid register */
-static inline bool is_register(int reg)
-{
-    return reg >= EXPR_REG_START && reg < REG_ENUM_LIMIT;
-}
-
 /*
  * REX flags
  */
@@ -555,22 +549,36 @@ static inline bool is_register(int reg)
 #define REX_V       0x0100  /* Instruction uses VEX/XOP instead of REX */
 #define REX_NH      0x0200  /* Instruction which doesn't use high regs */
 #define REX_EV      0x0400  /* Instruction uses EVEX instead of REX */
+#define REX_2       0x0800  /* Instruction requires REX2 */
+#define REX_B1      0x1000  /* REX2/EVEX B second bit */
+#define REX_X1      0x2000  /* REX2/EVEX X second bit */
+#define REX_R1      0x4000  /* REX2/EVEX R second bit */
+
+#define REX_BXR0    0x0007
+#define REX_BXR1    0x7000
+#define REX_BXR     0x7007
+
+#define REX_rB	    (REX_B | REX_B1 | REX_H | REX_P)
+#define REX_rX	    (REX_X | REX_X1 | REX_H | REX_P)
+#define REX_rR	    (REX_R | REX_R1 | REX_H | REX_P)
 
 /*
  * EVEX bit field
  */
-#define EVEX_P0MM       0x0f        /* EVEX P[3:0] : Opcode map           */
-#define EVEX_P0RP       0x10        /* EVEX P[4] : High-16 reg            */
-#define EVEX_P0X        0x40        /* EVEX P[6] : High-16 rm             */
-#define EVEX_P1PP       0x03        /* EVEX P[9:8] : Legacy prefix        */
-#define EVEX_P1VVVV     0x78        /* EVEX P[14:11] : NDS register       */
-#define EVEX_P1W        0x80        /* EVEX P[15] : Osize extension       */
-#define EVEX_P2AAA      0x07        /* EVEX P[18:16] : Embedded opmask    */
-#define EVEX_P2VP       0x08        /* EVEX P[19] : High-16 NDS reg       */
-#define EVEX_P2B        0x10        /* EVEX P[20] : Broadcast / RC / SAE  */
-#define EVEX_P2LL       0x60        /* EVEX P[22:21] : Vector length      */
+#define EVEX_P0MM       0x0f00      /* EVEX P[3:0] : Opcode map           */
+#define EVEX_P0RP       0x1000      /* EVEX P[4] : High-16 reg            */
+#define EVEX_P0X        0x4000      /* EVEX P[6] : High-16 rm             */
+#define EVEX_P1PP       0x030000    /* EVEX P[9:8] : Legacy prefix        */
+#define EVEX_P1VVVV     0x780000    /* EVEX P[14:11] : NDS register       */
+#define EVEX_P1W        0x800000    /* EVEX P[15] : Osize extension       */
+#define EVEX_P2AAA      0x07000000  /* EVEX P[18:16] : Embedded opmask    */
+#define EVEX_P2NF       0x04000000  /* EVEX P[18]: No flags bit           */
+#define EVEX_P2VP       0x08000000  /* EVEX P[19] : High-16 NDS reg       */
+#define EVEX_P2B        0x10000000  /* EVEX P[20] : Broadcast / RC / SAE  */
+#define EVEX_P2ND       EVEX_P2B    /* EVEX P[20] : New destination       */
+#define EVEX_P2LL       0x60000000  /* EVEX P[22:21] : Vector length      */
 #define EVEX_P2RC       EVEX_P2LL   /* EVEX P[22:21] : Rounding control   */
-#define EVEX_P2Z        0x80        /* EVEX P[23] : Zeroing/Merging       */
+#define EVEX_P2Z        0x80000000  /* EVEX P[23] : Zeroing/Merging       */
 
 /*
  * REX_V "classes" (prefixes which behave like VEX)
@@ -610,10 +618,12 @@ enum prefixes { /* instruction prefixes */
     P_BND,
     P_NOBND,
     P_REX,
+    P_REX2,
     P_EVEX,
     P_VEX,
     P_VEX3,
     P_VEX2,
+    P_NF,
     PREFIX_ENUM_LIMIT
 };
 
@@ -718,6 +728,7 @@ enum prefix_pos {
     PPS_OSIZE,          /* Operand size prefix */
     PPS_ASIZE,          /* Address size prefix */
     PPS_REX,            /* REX/VEX type */
+    PPS_NF,             /* No flags */
     MAXPREFIX           /* Total number of prefix slots */
 };
 
@@ -761,6 +772,7 @@ typedef struct insn { /* an instruction itself */
     enum opcode     opcode;                 /* the opcode - not just the string */
     int             operands;               /* how many operands? 0-7 (more if db et al) */
     int             addr_size;              /* address size */
+    int             op_size;                /* operand size */
     operand         oprs[MAX_OPERANDS];     /* the operands, defined as above */
     extop           *eops;                  /* extended operands */
     int             eops_float;             /* true if DD and floating */
@@ -768,11 +780,10 @@ typedef struct insn { /* an instruction itself */
     bool            forw_ref;               /* is there a forward reference? */
     bool            rex_done;               /* REX prefix emitted? */
     int             rex;                    /* Special REX Prefix */
-    int             vexreg;                 /* Register encoded in VEX prefix */
+    int             vexreg;                 /* Register encoded in VEX.V */
     int             vex_cm;                 /* Class and M field for VEX prefix */
     int             vex_wlp;                /* W, P and L information for VEX prefix */
-    uint8_t         evex_p[3];              /* EVEX.P0: [RXB,R',00,mm], P1: [W,vvvv,1,pp] */
-                                            /* EVEX.P2: [z,L'L,b,V',aaa] */
+    uint32_t	    evex;                   /* EVEX prefix under construction */
     enum ttypes     evex_tuple;             /* Tuple type for compressed Disp8*N */
     int             evex_rm;                /* static rounding mode for AVX512 (EVEX) */
     int8_t          evex_brerop;            /* BR/ER/SAE operand position */
diff --git a/include/nasmlib.h b/include/nasmlib.h
index 299734db8..472c8fd4a 100644
--- a/include/nasmlib.h
+++ b/include/nasmlib.h
@@ -456,7 +456,7 @@ static inline int64_t const_func sext(int64_t value, unsigned int bits)
     /* sext(foo,0) == sext(foo,1) */
     bits = bits ? 64-bits : 63;
 
-    return value >> bits << bits;
+    return value << bits >> bits;
 }
 
 /* Zero-extend a value to an arbitrary number of bits */
@@ -482,20 +482,20 @@ static inline uint64_t const_func zext(uint64_t value, unsigned int bits)
         return 0;
 
     bits = 64-bits;
-    return value >> bits << bits;
+    return value << bits >> bits;
 }
 
-static inline bool const_func overflow_signed(int64_t value, int bytes)
+static inline bool const_func overflow_signed(int64_t value, unsigned int bytes)
 {
     return sext(value, bytes << 3) != value;
 }
 
-static inline bool const_func overflow_unsigned(uint64_t value, int bytes)
+static inline bool const_func overflow_unsigned(uint64_t value, unsigned int bytes)
 {
     return zext(value, bytes << 3) != value;
 }
 
-static inline bool const_func overflow_general(int64_t value, int bytes)
+static inline bool const_func overflow_general(int64_t value, unsigned int bytes)
 {
     return overflow_unsigned(value, bytes) && overflow_signed(value, bytes);
 }
diff --git a/include/opflags.h b/include/opflags.h
index ba87af7c8..2aa291f61 100644
--- a/include/opflags.h
+++ b/include/opflags.h
@@ -178,16 +178,23 @@
 #define REG_CLASS_BND           GEN_REG_CLASS(9)
 #define REG_CLASS_RM_TMM	GEN_REG_CLASS(10)
 
+/* Verify value to be a valid register */
+static inline bool is_register(opflags_t reg)
+{
+    return reg >= EXPR_REG_START && reg < REG_ENUM_LIMIT;
+}
+
 static inline bool is_class(opflags_t class, opflags_t op)
 {
-	return !(class & ~op);
+    return !(class & ~op);
 }
 
 static inline bool is_reg_class(opflags_t class, opflags_t reg)
 {
-	if (reg >= EXPR_REG_START && reg <= EXPR_REG_END)
-		return is_class(class, nasm_reg_flags[reg]);
-	return false;
+    if (!is_register(reg))
+        return false;
+
+    return is_class(class, nasm_reg_flags[reg]);
 }
 
 #define IS_SREG(reg)                is_reg_class(REG_SREG, (reg))
diff --git a/test/apx.asm b/test/apx.asm
new file mode 100644
index 000000000..0914bf165
--- /dev/null
+++ b/test/apx.asm
@@ -0,0 +1,33 @@
+	bits 64
+	ccmpnz {dfv=} rax,rbx
+	ccmpnz {dfv=cf} rax,rbx
+	ccmpnz {dfv=zf} rax,rbx
+	ccmpnz {dfv=sf} rax,rbx
+	ccmpnz {dfv=of} rax,rbx
+	ccmpnz {dfv=of,cf} rax,rbx
+	ccmpnz {dfv=of,cf} [rax],rbx
+	ccmpnz {dfv=of,cf} dword [rax],3
+	ccmpnz {dfv=of,cf} dword [r31],3
+	ccmpnz 15, dword [r31], 3
+
+	ccmpnz {dfv=of,cf} dword [r31], byte 3
+	ccmpnz {dfv=of,cf} dword [r31], 3
+	ccmpnz {dfv=of,cf} dword [r31], dword 3
+	ccmpnz {dfv=of,cf} dword [r31], strict dword 3
+	ccmpnz {dfv=of,cf} dword [r31], 0xaabbccdd
+	ccmpnz {dfv=of,cf} dword [r31], dword 0xaabbccdd
+	ccmpnz {dfv=of,cf} qword [r31], 0xaabbccdd
+
+	push rax
+	pushp rax
+	push rax, rbx
+	push rax:rbx
+
+	pop rax:rbx
+	pop rbx, rax
+
+	add      al,[rdx],cl
+	add {nf} al,[rdx],cl
+	add      [rdx],cl
+	add {evex} [rdx],cl
+	add {nf} [rdx],cl
diff --git a/x86/bytecode.txt b/x86/bytecode.txt
index 1b5d5c62a..4c9996275 100644
--- a/x86/bytecode.txt
+++ b/x86/bytecode.txt
@@ -47,8 +47,8 @@ Codes            Mnemonic        Explanation
                                          an arbitrary value in bits 3..0 (assembled as zero.)
 \2ab             /b                      a ModRM, calculated on EA in operand a, with the reg
                                          field equal to digit b.
-\240..\247                               this instruction uses EVEX rather than REX or VEX/XOP, with the
-                                         V register number taken from operand 0..7 (which may
+\240..\243                               this instruction uses EVEX rather than REX or VEX/XOP, with the
+                                         V register number taken from operand "b" (0..3) (which may
 					 be an immediate, as is used for DFV.)
 \250                                     this instruction uses EVEX rather than REX or VEX/XOP, with the
                                          V register number set to 0 (subject to the XOR as defined
@@ -74,10 +74,10 @@ EVEX prefixes are followed by the sequence:
                    (compressed displacement encoding)
 
 \254..\257       id,s                    a signed 32-bit operand to be extended to 64 bits.
-\260..\267                               this instruction uses VEX/XOP rather than REX, with the
-                                         V field taken from operand 0..7.
+\260..\263                               this instruction uses VEX/XOP rather than REX, with the
+                                         V register taken from operand "b" 0..3.
 \270                                     this instruction uses VEX/XOP rather than REX, with the
-                                         V field set to 1111b.
+                                         V register set to 0.
 VEX/XOP prefixes are followed by the sequence:
 \tmm\wlp	tmm format:	tt 0mm mmm
 		[vex] tt = 0
@@ -85,15 +85,15 @@ VEX/XOP prefixes are followed by the sequence:
 
 		mmmmm = M field
 
-		wlp format:	00 0w0 lpp
+		wlp format:	w0 00l lpp
                 [l0]  ll = 0 for L = 0 (.128, .lz)
                 [l1]  ll = 1 for L = 1 (.256)
                 [lig] ll = 0 for L don't care (always assembled as 0) with IF_LIG
 
-                [w0]  ww = 0 for W = 0
-                [w1 ] ww = 1 for W = 1
-                [wig] ww = 0 for W don't care (always assembled as 0) with IF_WIG
-                [ww]  ww = 0 for W used as REX.W with IF_WW
+                [w0]  w = 0 for W = 0
+                [w1 ] w = 1 for W = 1
+                [wig] w = 0 for W don't care (always assembled as 0) with IF_WIG
+                [ww]  w = 0 for W used as REX.W with IF_WW
 
 t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 
@@ -130,10 +130,11 @@ t = 0 for VEX (C4/C5), t = 1 for XOP (8F).
 \340             resb                        reserve <operand 0> bytes of uninitialized storage.
                                              Operand 0 had better be a segmentless constant.
 \341             wait                        this instruction needs a WAIT "prefix"
-\344..\347	 rexx[.0f|m1][.w]	     REX or REX2 prefix, bits [1:0] set the M or W bits (promotable)
+\344..\347	 rexx[.0f|m1][.w]	     REX or REX2 prefix, bits [1:0] set the W or M bits (promotable)
 					     rexx.m1 -> rex 0f if REX2 is not generated.
 					     The W bit will also be set if o64 is specified.
-35x		 rex2[.0f|m1][.x1][.w]	     obligatory REX2 prefix, bits [2:0] set the X1, M, and W bits
+					     The REX prefix can be omitted using normal REX omission rules.
+\35x		 rex2[.0f|m1][.x1][.w]	     obligatory REX2 prefix, bits [2:0] set the X1, W, and M bits
 					     The W bit will also be set if o64 is specified.
 \360             np                          no SSE prefix (== \364\331)
 \361                                         66 SSE prefix (== \366\331)
diff --git a/x86/disp8.c b/x86/disp8.c
index 64c921b61..b99db7ee6 100644
--- a/x86/disp8.c
+++ b/x86/disp8.c
@@ -47,10 +47,10 @@ uint8_t get_disp8N(insn *ins)
     static const uint8_t hv_n[2][VLMAX]    =  {{8, 16, 32}, {4, 4, 4}};
     static const uint8_t dup_n[VLMAX]      =   {8, 32, 64};
 
-    bool evex_b           = (ins->evex_p[2] & EVEX_P2B) >> 4;
+    bool evex_b           = !!(ins->evex & EVEX_P2B);
     enum ttypes   tuple   = ins->evex_tuple;
-    enum vectlens vectlen = (ins->evex_p[2] & EVEX_P2LL) >> 5;
-    bool evex_w           = (ins->evex_p[1] & EVEX_P1W) >> 7;
+    enum vectlens vectlen = (ins->evex & EVEX_P2LL) >> 29;
+    bool evex_w           = !!(ins->evex & EVEX_P1W);
     uint8_t n = 0;
 
     switch(tuple) {
diff --git a/x86/insns.dat b/x86/insns.dat
index 8061d4469..dde5ef3a2 100644
--- a/x86/insns.dat
+++ b/x86/insns.dat
@@ -5988,7 +5988,7 @@ VPSHRDD		zmmreg|mask|z,zmmreg*,zmmrm512|b32,imm8	[rvmi:fv:  evex.nds.512.66.0f3a
 VPSHRDQ		xmmreg|mask|z,xmmreg*,xmmrm128|b64,imm8	[rvmi:fv:  evex.nds.128.66.0f3a.w1 73 /r ib]	AVX512VBMI2,AVX512VL,FUTURE
 VPSHRDQ		ymmreg|mask|z,ymmreg*,ymmrm256|b64,imm8	[rvmi:fv:  evex.nds.256.66.0f3a.w1 73 /r ib]	AVX512VBMI2,AVX512VL,FUTURE
 VPSHRDQ		zmmreg|mask|z,zmmreg*,zmmrm512|b64,imm8	[rvmi:fv:  evex.nds.512.66.0f3a.w1 73 /r ib]	AVX512VBMI2,FUTURE
-VPSHRDVW	xmmreg|mask|z,xmmreg*,xmmrm128    	[rvmi:fvm: evex.dds.128.66.0f38.w1 72 /r]	AVX512VBMI2,AVX512VL,FUTURE
+VPSHRDVW	xmmreg|mask|z,xmmreg*,xmmrm128		[rvmi:fvm: evex.dds.128.66.0f38.w1 72 /r]	AVX512VBMI2,AVX512VL,FUTURE
 VPSHRDVW	ymmreg|mask|z,ymmreg*,ymmrm256		[rvmi:fvm: evex.dds.256.66.0f38.w1 72 /r]	AVX512VBMI2,AVX512VL,FUTURE
 VPSHRDVW	zmmreg|mask|z,zmmreg*,zmmrm512		[rvmi:fvm: evex.dds.512.66.0f38.w1 72 /r]	AVX512VBMI2,FUTURE
 VPSHRDVD	xmmreg|mask|z,xmmreg*,xmmrm128|b32	[rvmi:fv:  evex.dds.128.66.0f38.w0 73 /r]	AVX512VBMI2,AVX512VL,FUTURE
@@ -6394,43 +6394,48 @@ HRESET		imm,reg_eax				[i-:	f3 0f 3a f0 c0 ib	]	HRESET,FUTURE,PRIV,SB
 HRESET		imm					[i:	f3 0f 3a f0 c0 ib	]	HRESET,FUTURE,PRIV,SB,ND
 
 ;# APX test
-CCMPscc		spec4,rm8,reg8			[vmr:	evex.scc.dfv.l0.np.m4.wig 38 /r	]	SM23,APX
-CCMPscc		spec4,rm16,reg16		[vmr:	evex.scc.dfv.l0.66.m4.w0  39 /r	]	SM23,APX
-CCMPscc		spec4,rm32,reg32		[vmr:	evex.scc.dfv.l0.np.m4.w0  39 /r	]	SM23,APX
-CCMPscc		spec4,rm64,reg64		[vmr:	evex.scc.dfv.l0.np.m4.w1  39 /r	]	SM23,APX
+CCMPscc		spec4,rm8,reg8			[vmr:	    evex.scc.dfv.l0.np.m4.wig 38 /r	]	APX,SM23
+CCMPscc		spec4,rm16,reg16		[vmr:	o16 evex.scc.dfv.l0.66.m4.w0  39 /r	]	APX,SM23
+CCMPscc		spec4,rm32,reg32		[vmr:	o32 evex.scc.dfv.l0.np.m4.w0  39 /r	]	APX,SM23
+CCMPscc		spec4,rm64,reg64		[vmr:	o64 evex.scc.dfv.l0.np.m4.w1  39 /r	]	APX,SM23
 
-CCMPscc		spec4,reg8,rm8			[vrm:	evex.scc.dfv.l0.np.m4.wig 3a /r	]	SM23,APX
-CCMPscc		spec4,reg16,rm16		[vrm:	evex.scc.dfv.l0.66.m4.w0  3b /r	]	SM23,APX
-CCMPscc		spec4,reg32,rm32		[vrm:	evex.scc.dfv.l0.np.m4.w0  3b /r	]	SM23,APX
-CCMPscc		spec4,reg64,rm64		[vrm:	evex.scc.dfv.l0.np.m4.w1  3b /r	]	SM23,APX
+CCMPscc		spec4,reg8,rm8			[vrm:	    evex.scc.dfv.l0.np.m4.wig 3a /r	]	APX,SM23
+CCMPscc		spec4,reg16,rm16		[vrm:	o16 evex.scc.dfv.l0.66.m4.w0  3b /r	]	APX,SM23
+CCMPscc		spec4,reg32,rm32		[vrm:	o32 evex.scc.dfv.l0.np.m4.w0  3b /r	]	APX,SM23
+CCMPscc		spec4,reg64,rm64		[vrm:	o64 evex.scc.dfv.l0.np.m4.w1  3b /r	]	APX,SM23
 
-CCMPscc		spec4,rm8,imm			[vmi:	evex.scc.dfv.l0.np.m4.wig 80 /7	]	SM23,APX
-CCMPscc		spec4,rm16,sbyteword		[vmi:	evex.scc.dfv.l0.66.m4.w0  83 /7	]	SM23,APX
-CCMPscc		spec4,rm32,sbytedword		[vmi:	evex.scc.dfv.l0.np.m4.w0  83 /7	]	SM23,APX
-CCMPscc		spec4,rm64,sbytedword		[vmi:	evex.scc.dfv.l0.np.m4.w1  83 /7	]	SM23,APX
-CCMPscc		spec4,rm16,imm			[vmi:	evex.scc.dfv.l0.66.m4.w0  81 /7	]	SM23,APX
-CCMPscc		spec4,rm32,imm			[vmi:	evex.scc.dfv.l0.np.m4.w0  81 /7	]	SM23,APX
-CCMPscc		spec4,rm64,imm			[vmi:	evex.scc.dfv.l0.np.m4.w1  81 /7	]	SM23,APX
+CCMPscc		spec4,rm8,imm			[vmi:	    evex.scc.dfv.l0.np.m4.wig 80 /7	ib   ]	APX,SM23
+CCMPscc		spec4,rm16,imm8			[vmi:	o16 evex.scc.dfv.l0.66.m4.w0  83 /7	ib,s ]	APX
+CCMPscc		spec4,rm16,sbyteword		[vmi:	o16 evex.scc.dfv.l0.66.m4.w0  83 /7	ib,s ]	APX,SM23,ND
+CCMPscc		spec4,rm16,imm			[vmi:	o16 evex.scc.dfv.l0.66.m4.w0  81 /7	iw   ]	APX,SM23
+CCMPscc		spec4,rm32,imm8			[vmi:	o32 evex.scc.dfv.l0.np.m4.w0  83 /7	ib,s ]	APX
+CCMPscc		spec4,rm32,sbytedword		[vmi:	o32 evex.scc.dfv.l0.np.m4.w0  83 /7	ib,s ]	APX,SM23,ND
+CCMPscc		spec4,rm32,imm			[vmi:	o32 evex.scc.dfv.l0.np.m4.w0  81 /7	id   ]	APX,SM23
+CCMPscc		spec4,rm64,imm8			[vmi:	o64 evex.scc.dfv.l0.np.m4.w1  83 /7	ib,s ]	APX
+CCMPscc		spec4,rm64,sbytedword		[vmi:	o64 evex.scc.dfv.l0.np.m4.w1  83 /7	ib,s ]	APX,SM23,ND
+CCMPscc		spec4,rm64,imm			[vmi:	o64 evex.scc.dfv.l0.np.m4.w1  81 /7	id,s ]	APX,SM23
 
-PUSH2		reg64,reg64			[vm:	evex.nd.l0.np.m4.w0	  ff /6	]	APX
-PUSH		reg64,reg64			[vm:	evex.nd.l0.np.m4.w0	  ff /6	]	APX,ND
-PUSH2		reg64:reg64			[vm:	evex.nd.l0.np.m4.w0	  ff /6	]	APX,ND
-PUSH		reg64:reg64			[vm:	evex.nd.l0.np.m4.w0	  ff /6	]	APX,ND
-PUSH2P		reg64,reg64			[vm:	evex.nd.l0.np.m4.w1	  ff /6	]	APX
-PUSHP		reg64,reg64			[vm:	evex.nd.l0.np.m4.w1	  ff /6	]	APX,ND
-PUSH2P		reg64:reg64			[vm:	evex.nd.l0.np.m4.w1	  ff /6	]	APX,ND
-PUSHP		reg64:reg64			[vm:	evex.nd.l0.np.m4.w1	  ff /6	]	APX,ND
-POP2		reg64,reg64			[vm:	evex.nd.l0.np.m4.w0	  8f /0	]	APX
-POP		reg64,reg64			[vm:	evex.nd.l0.np.m4.w0	  8f /0	]	APX,ND
-POP2		reg64:reg64			[mv:	evex.nd.l0.np.m4.w0	  8f /0	]	APX,ND
-POP		reg64:reg64			[mv:	evex.nd.l0.np.m4.w0	  8f /0	]	APX,ND
-POP2P		reg64,reg64			[vm:	evex.nd.l0.np.m4.w1	  8f /0	]	APX
-POPP		reg64,reg64			[vm:	evex.nd.l0.np.m4.w1	  8f /0	]	APX,ND
-POP2P		reg64:reg64			[mv:	evex.nd.l0.np.m4.w1	  8f /0	]	APX,ND
-POPP		reg64:reg64			[mv:	evex.nd.l0.np.m4.w1	  8f /0	]	APX,ND
+PUSH2		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0	  ff /6	]	APX
+PUSH		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0        ff /6	]	APX,ND
+PUSH2		reg64:reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0	  ff /6	]	APX,ND
+PUSH		reg64:reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0	  ff /6	]	APX,ND
+PUSH2P		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  ff /6	]	APX
+PUSHP		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  ff /6	]	APX,ND
+PUSH2P		reg64:reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  ff /6	]	APX,ND
+PUSHP		reg64:reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  ff /6	]	APX,ND
+POP2		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0	  8f /0	]	APX
+POP		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w0	  8f /0	]	APX,ND
+POP2		reg64:reg64			[mv:	o64nw evex.nd1.l0.np.m4.w0	  8f /0	]	APX,ND
+POP		reg64:reg64			[mv:	o64nw evex.nd1.l0.np.m4.w0	  8f /0	]	APX,ND
+POP2P		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  8f /0	]	APX
+POPP		reg64,reg64			[vm:	o64nw evex.nd1.l0.np.m4.w1	  8f /0	]	APX,ND
+POP2P		reg64:reg64			[mv:	o64nw evex.nd1.l0.np.m4.w1	  8f /0	]	APX,ND
+POPP		reg64:reg64			[mv:	o64nw evex.nd1.l0.np.m4.w1	  8f /0	]	APX,ND
 
-PUSHP		reg64				[r:	rex2.m0.w1 50+r			]	APX
-POPP		reg64				[r:	rex2.m0.w1 58+r			]	APX
+PUSHP		reg64				[r:	o64nw rex2.m0.w1 50+r		]	APX
+POPP		reg64				[r:	o64nw rex2.m0.w1 58+r		]	APX
+
+ADD		reg8?,rm8,reg8			[vmr:	evex.ndx.l0.np.m4.wig 00 /r	]	APX,NF
 
 ;# Systematic names for the hinting nop instructions
 ; These should be last in the file
diff --git a/x86/insns.pl b/x86/insns.pl
index 55092dff8..0690c5622 100755
--- a/x86/insns.pl
+++ b/x86/insns.pl
@@ -77,19 +77,26 @@ sub xpush($@) {
 }
 
 # Generate relaxed form patterns if applicable
+# * is used for an optional source operand, duplicating the previous one
+#   in the encoding if it is missing.
+# ? is used for an optional destination operand which is not encoded if
+#   missing; if combined evex.ndx set the nd bit if present.
 sub relaxed_forms(@) {
     my @field_list = @_;
 
     foreach my $fields (@_) {
-	next unless ($fields->[1] =~ /\*/);
+	next unless ($fields->[1] =~ /[\*\?]/);
 
 	# This instruction has relaxed form(s)
-	if ($fields->[2] !~ /^\[/) {
+	if ($fields->[2] !~ /^(\[\s*)(\S+)(\s*:.*\])$/) {
 	    warn "$fname:$line: has an * operand but uses raw bytecodes\n";
 	    next;
 	}
+	my @f2o = ($1, $2, $3);
 
 	my $opmask = 0;
+	my $ndmask = 0;
+	my $ndflag = 0;
 	my @ops = split(/,/, $fields->[1]);
 	for (my $oi = 0; $oi < scalar @ops; $oi++) {
 	    if ($ops[$oi] =~ /\*$/) {
@@ -98,22 +105,46 @@ sub relaxed_forms(@) {
 		    next;
 		}
 		$opmask |= 1 << $oi;
+	    } elsif ($ops[$oi] =~ /\?$/) {
+		$opmask |= 1 << $oi;
+		$ndmask |= 1 << $oi;
+		$ndflag = 1;
 	    }
 	}
 
+	# If .ndx is present, then change it to .nd0 or .nd1
+	# Set to .nd0 if no ndmask fields are present, otherwise 1
+	$fields->[2] =~ s/(\.nd)x\b/$1$ndflag/;
+
 	for (my $oi = 1; $oi < (1 << scalar @ops); $oi++) {
 	    if (($oi & ~$opmask) == 0) {
 		my @xops = ();
-		my $omask = ~$oi;
-		for ($oj = 0; $oj < scalar(@ops); $oj++) {
-		    if ($omask & 1) {
-			push(@xops, $ops[$oj]);
+		my $ndflag = 0;
+		my $relax = 0;
+		my $rbit = 1;
+		my $ondflag = $ndflag;
+		my $f2 = $f2o[0];
+		for (my $oj = 0; $oj < scalar(@ops); $oj++) {
+		    my $ob = 1 << $oj;
+		    if ($ob & $ndmask & $oi) {
+			# Make it disappear completely
+			$ondflag = 0;
+		    } else {
+			if ($ob & ~$oi) {
+			    push(@xops, $ops[$oj]);
+			} else {
+			    $relax |= $rbit;
+			}
+			$rbit <<= 1;
+			$f2 .= substr($f2o[1], $oj, 1);
 		    }
-		    $omask >>= 1;
 		}
+		$f2 .= $f2o[2];
 		my @ff = @$fields;
 		$ff[1] = join(',', @xops);
-		$ff[4] = $oi;
+		$f2 =~ s/(\.nd)x\b/$1$ondflag/;
+		$ff[2] = $f2;
+		$ff[4] = $relax;
 		push(@field_list, [@ff]);
 	    }
 	}
@@ -528,7 +559,7 @@ sub format_insn($$$$$) {
     return (undef, undef) if $operands eq "ignore";
 
     # format the operands
-    $operands =~ s/\*//g;
+    $operands =~ s/[\*\?]//g;
     $operands =~ s/:/|colon,/g;
     @ops = ();
     @opsize = ();
@@ -615,6 +646,11 @@ sub format_insn($$$$$) {
 	}
     }
 
+    # Flags that imply long mode only
+    if ($flags{'APX'}) {
+	$flags->{'LONG'}++;
+    }
+
     # Look for SM flags clearly inconsistent with operand bitsizes
     if ($flags{'SM'} || $flags{'SM2'} || $flags{'SM23'}) {
 	my $ssize = 0;
@@ -700,10 +736,10 @@ sub show_bytecodes($) {
 		if ($c <= 4) {
 		    $literals = $c;
 		    $hexlit = 1;
-		} elsif ($c >= 0240 && $c <= 0250) {
+		} elsif (($c & 3) == 0240 || $c == 0250) {
 		    $literals = 3;
 		    $hexlit = 1;
-		} elsif ($c >= 0260 && $c <= 0270) {
+		} elsif (($c & 3) == 0260 || $c == 0270) {
 		    $literals = 2;
 		    $hexlit = 0;
 		} elsif ($c == 0171) {
@@ -835,17 +871,24 @@ sub startseq($$) {
             return addprefix($prefix, $c1, $c1|2);
         } elsif ($c0 == 0 || $c0 == 0340) {
             return $prefix;
-        } elsif (($c0 & ~3) == 0260 || $c0 == 0270 ||
-                 ($c0 & ~3) == 0240 || $c0 == 0250) {
+        } elsif (($c0 & ~3) == 0260 || $c0 == 0270) {
+	    # VEX/XOP
             my($c,$m,$wlp);
             $m   = shift(@codes);
             $wlp = shift(@codes);
             $c = ($m >> 6);
             $m = $m & 31;
             $prefix .= sprintf('%s%02X%01X', $vex_class[$c], $m, $wlp & 3);
-            if ($c0 < 0260) {
-                my $tuple = shift(@codes);
-            }
+	} elsif (($c0 & ~3) == 0260 || $c0 == 0270) {
+	    # EVEX
+	    my @p;
+	    push(@p, shift(@codes));
+	    push(@p, shift(@codes));
+	    push(@p, shift(@codes));
+	    my $tuple = shift(@codes);
+	    my $m = $p[0] & 7;
+	    my $p = $p[1] & 3;
+            $prefix .= sprintf('%s%02X%01X', 'evex', $m, $p);
         } elsif ($c0 >= 0172 && $c0 <= 173) {
             shift(@codes);      # Skip is4 control byte
         } else {
@@ -918,6 +961,8 @@ sub byte_code_compile($$$) {
     my ($op, $oq);
     my $opex;
 
+    printf STDERR "%s (%x) %s\n", $str, $relax, join(',', sort(keys(%$flags)));
+
     my %imm_codes = (
         'ib'        => 020,     # imm8
         'ib,u'      => 024,     # Unsigned imm8
@@ -1063,7 +1108,7 @@ sub byte_code_compile($$$) {
         } elsif ($op =~ /^(vex|xop)(|\..*)$/) {
             my $vexname = $1;
             my $c = $vexmap{$vexname};
-            my ($m,$w,$l,$p) = (undef,2,undef,0);
+            my ($m,$w,$l,$p) = (undef,undef,undef,0);
             my $has_nds = 0;
             my @subops = split(/\./, $2);
 	    foreach $oq (@subops) {
@@ -1111,15 +1156,22 @@ sub byte_code_compile($$$) {
 		    die "$fname:$line: undefined modifier: $vexname.$oq\n";
 		}
 	    }
-            if (!defined($m) || !defined($w) || !defined($l) || !defined($p)) {
+            if (!defined($m) || !defined($l) || !defined($p)) {
                 die "$fname:$line: missing fields in \U$vexname\E specification\n";
             }
+
+	    if (!defined($w)) {
+		$w = 0;
+		$flags->{'WIG'}++;
+	    }
+
 	    my $minmap = ($c == 1) ? 8 : 0; # 0-31 for VEX, 8-31 for XOP
 	    if ($m < $minmap || $m > 31) {
 		die "$fname:$line: Only maps ${minmap}-31 are valid for \U${vexname}\n";
 	    }
+	    push(@codes, 05) if ($oppos{'v'} > 3);
             push(@codes, defined($oppos{'v'}) ? 0260+$oppos{'v'} : 0270,
-                 ($c << 6)+$m, ($w << 4)+($l << 2)+$p);
+                 ($c << 6)+$m, ($w << 7)+($l << 2)+$p);
 
 	    $flags->{'VEX'}++;
             $prefix_ok = 0;
@@ -1138,6 +1190,9 @@ sub byte_code_compile($$$) {
 		    $l = 1;
 		} elsif ($oq eq '512' || $oq eq 'l2') {
 		    $l = 2;
+		} elsif ($oq eq '1024' || $oq eq '1k' || $oq eq 'l3') {
+		    # Not actually defined, but...
+		    $l = 3;
 		} elsif ($oq eq 'w0') {
 		    $w = 0;
 		} elsif ($oq eq 'w1') {
@@ -1181,12 +1236,14 @@ sub byte_code_compile($$$) {
 		    $flags->{'DFV'}++;
 		    $dfv = 1;
 		    push(@bad_op, ['v', $oq]);
+		} elsif ($oq =~ /^nd([01])$/) {
+		    $nd = $1 + 0;
 		} elsif ($oq =~ /^(nds|ndd|nd|dds)$/) {
 		    if (!defined($oppos{'v'})) {
 			die "$fname:$line: evex.$oq without 'v' operand\n";
 		    }
 		    $nds = 1;
-		    $ndd = $oq eq 'nd';
+		    $nd  = $oq eq 'nd';
 		} else {
 		    die "$fname:$line: undefined modifier: evex.$oq\n";
 		}
@@ -1216,7 +1273,9 @@ sub byte_code_compile($$$) {
 	    $p[2] |= 0x04 if ($nf);
 	    $p[2] |= 0x10 if ($nd);
 
+	    push(@codes, 05) if ($oppos{'v'} > 3);
 	    push(@codes, defined($oppos{'v'}) ? 0240+$oppos{'v'} : 0250, @p);
+	    push(@codes, $tup);
 
 	    $flags->{'EVEX'}++;
             $prefix_ok = 0;
@@ -1242,8 +1301,11 @@ sub byte_code_compile($$$) {
 	    }
 	    push(@codes, $oq);
 
-	    $flags->{'REX2'}++ if ($rex2);
-	    $flags->{'REXX'}++;
+	    if ($rex2) {
+		$flags->{'APX'}++;
+		$flags->{'LONG'}++;
+	    }
+	    $flags->{'REX2'}++;
             $prefix_ok = 0;
         } elsif (defined $imm_codes{$op}) {
             if ($op eq 'seg') {