Commit 40a6c128be for qemu.org

commit 40a6c128beeb72fa6d17e8c393b3d60e09ca0a69
Author: Zixin Wang <w1355457260@gmail.com>
Date:   Tue Nov 10 18:10:19 2026 +0800

    target/i386/tcg: treat ENTER's size operand as unsigned

    The first immediate operand of ENTER is the size of the dynamic storage
    area and is unsigned.  Decoding it as a signed immediate sign-extends
    values with bit 15 set, moving the stack pointer in the wrong direction.

    Decode it with I_unsigned and add a regression test covering 32-bit and
    64-bit guests.

    Tested with a minimal x86_64-linux-user build; the new test fails before
    the change and passes after it.

    Resolves: https://gitlab.com/qemu-project/qemu/-/issues/4643
    Signed-off-by: Zixin Wang <w1355457260@gmail.com>
    Link: https://lore.kernel.org/r/CABG6VKODisrETOuH5thhimg_DtJLH+1wmARKL5OsoND5Qse_TA@mail.gmail.com
    Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/target/i386/tcg/decode-new.c.inc b/target/i386/tcg/decode-new.c.inc
index 19c76a1200..bd9a94b316 100644
--- a/target/i386/tcg/decode-new.c.inc
+++ b/target/i386/tcg/decode-new.c.inc
@@ -1972,7 +1972,7 @@ static const X86OpEntry opcodes_root[256] = {
     [0xBE] = X86_OP_ENTRYwr(MOV, LoBits,v, I,v),
     [0xBF] = X86_OP_ENTRYwr(MOV, LoBits,v, I,v),

-    [0xC8] = X86_OP_ENTRYrr(ENTER, I,w, I,b),
+    [0xC8] = X86_OP_ENTRYrr(ENTER, I_unsigned, w, I, b),
     [0xC9] = X86_OP_ENTRY0(LEAVE),
     [0xCA] = X86_OP_ENTRYr(RETF,  I,w),
     [0xCB] = X86_OP_ENTRY0(RETF),
diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c
index d4ccddc33f..719771032f 100644
--- a/target/i386/tcg/translate.c
+++ b/target/i386/tcg/translate.c
@@ -1989,7 +1989,7 @@ static void gen_popa(DisasContext *s)
     gen_stack_update(s, 8 * size);
 }

-static void gen_enter(DisasContext *s, int esp_addend, int level)
+static void gen_enter(DisasContext *s, uint16_t esp_addend, int level)
 {
     MemOp d_ot = mo_pushpop(s, s->dflag);
     MemOp a_ot = mo_stacksize(s);
diff --git a/tests/tcg/i386/meson.build b/tests/tcg/i386/meson.build
index c912f37fab..d2ba841041 100644
--- a/tests/tcg/i386/meson.build
+++ b/tests/tcg/i386/meson.build
@@ -37,6 +37,7 @@ tests += {
                '-lm', '-fno-pie', '-Wl,--no-warn-rwx-segments'],
   },
   'test-idiv-overflow.c': {},
+  'test-i386-enter.c': {'cflags': ['-O2', '-fomit-frame-pointer']},
   '../multiarch/sha512.c': {
      'exe_name': 'sha512-sse',
      'cflags': ['-msse4.1', '-O3'],
diff --git a/tests/tcg/i386/test-i386-enter.c b/tests/tcg/i386/test-i386-enter.c
new file mode 100644
index 0000000000..509740d88c
--- /dev/null
+++ b/tests/tcg/i386/test-i386-enter.c
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * ENTER's 16-bit size operand is unsigned.  If the decoder sign-extends
+ * it, the stack pointer moves in the wrong direction.
+ */
+#include <stdio.h>
+
+static int check_enter(void)
+{
+    unsigned long before, after;
+
+#ifdef __x86_64__
+    __asm__ volatile(
+        "movq %%rsp, %0\n\t"
+        "enter $0x8000, $0\n\t"
+        "movq %%rsp, %1\n\t"
+        "leave\n\t"
+        : "=r"(before), "=r"(after)
+        :
+        : "memory");
+    return (before - after) == 0x8008 ? 0 : 1;
+#else
+    __asm__ volatile(
+        "movl %%esp, %0\n\t"
+        "enter $0x8000, $0\n\t"
+        "movl %%esp, %1\n\t"
+        "leave\n\t"
+        : "=r"(before), "=r"(after)
+        :
+        : "memory");
+    return (before - after) == 0x8004 ? 0 : 1;
+#endif
+}
+
+int main(void)
+{
+    if (check_enter()) {
+        printf("FAIL: ENTER treated the 16-bit size operand as signed\n");
+        return 1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/x86_64/meson.build b/tests/tcg/x86_64/meson.build
index 9738d04220..1efb35cf2e 100644
--- a/tests/tcg/x86_64/meson.build
+++ b/tests/tcg/x86_64/meson.build
@@ -50,6 +50,7 @@ tests += {
   '../i386/test-i386-blendv.c': {},
   '../i386/test-i386-bmi2.c': {'cflags': ['-O2', '-fwrapv']},
   '../i386/test-idiv-overflow.c': {},
+  '../i386/test-i386-enter.c': {'cflags': ['-O2', '-fomit-frame-pointer']},
 }

 tests += {