Add a default-error warning for impossible o64, a64, or a16 prefixes

The o64, a64 and a16 prefixes are only possible in specific
modes. It is a lot easier to debug with a more sensible error message.

Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

Author: H. Peter Anvin (Intel)

asm/assemble.c

@@ -1403,6 +1403,25 @@ static int64_t calcsize(insn *ins, const struct itemplate * const temp)
     /* Default operand size */
     ins->op_size = bits != 16 ? 32 : 16;
 
+    if (bits == 64) {
+        if (ins->prefixes[PPS_ASIZE] == P_A16) {
+            nasm_warn(WARN_PREFIX_BADMODE_A16,
+                      "a64 prefix invalid in 64-bit mode");
+            ins->prefixes[PPS_ASIZE] = 0;
+        }
+    } else {
+        if (ins->prefixes[PPS_OSIZE] == P_O64) {
+            nasm_warn(WARN_PREFIX_BADMODE_O64,
+                      "o64 prefix invalid in %d-bit mode", bits);
+            ins->prefixes[PPS_OSIZE] = P_none;
+        }
+        if (ins->prefixes[PPS_ASIZE] == P_A64) {
+            nasm_warn(WARN_PREFIX_BADMODE_A64,
+                      "a64 prefix invalid in %d-bit mode", bits);
+            ins->prefixes[PPS_ASIZE] = P_none;
+        }
+    }
+
     nasm_zero(need_pfx);
 
     while (*codes) {

asm/warnings.dat

@@ -254,6 +254,21 @@ pragma-unknown [off] unknown \c{%pragma} facility or directive
     Warns about an unknown \c{%pragma} directive.
     This is not yet implemented for most cases.
 
+prefix-badmode-a64 [err] a64 prefix invalid in 16/32-bit mode
+    Warns that an \c{a64} prefix was specified in 16- or 32-bit
+    mode. If the error is demoted to a warning or suppressed, the
+    prefix is ignored by the assembler.
+
+prefix-badmode-o64 [err] o64 prefix invalid in 16/32-bit mode
+    Warns that an \c{a64} prefix was specified in 16- or 32-bit
+    mode. If the error is demoted to a warning or suppressed, the
+    prefix is ignored by the assembler.
+
+prefix-badmode-a16 [err] a16 prefix invalid in 64-bit mode
+    Warns that an \c{a16} prefix was specified in 64-bit mode.
+    If the error is demoted to a warning or suppressed, the
+    prefix is ignored by the assembler.
+
 prefix-bnd [on] invalid \c{BND} prefix
 =bnd
     Warns about ineffective use of the \c{BND} prefix when the

doc/changes.src

@@ -53,6 +53,12 @@ It is the production version of NASM since 2025.
 
 \b Hopefully fix building with OpenWatcom.
 
+\b Generate a warning, promoted to error by default, on the use of
+   \c{a64} or \c{o64} prefixes in 16- or 32-bit mode or \c{a16}
+   prefixes in 64-bit mode. Those prefixes are not encodable; if
+   demoted to a warning or suppressed the prefix is ignored, but
+   likely will generate additional, harder to debug, error messages.
+
 
 \S{cl-3.00} Version 3.00