Commit 40a6c128be for qemu.org
commit 40a6c128beeb72fa6d17e8c393b3d60e09ca0a69
Author: Zixin Wang <w1355457260@gmail.com>
Date: Tue Nov 10 18:10:19 2026 +0800
target/i386/tcg: treat ENTER's size operand as unsigned
The first immediate operand of ENTER is the size of the dynamic storage
area and is unsigned. Decoding it as a signed immediate sign-extends
values with bit 15 set, moving the stack pointer in the wrong direction.
Decode it with I_unsigned and add a regression test covering 32-bit and
64-bit guests.
Tested with a minimal x86_64-linux-user build; the new test fails before
the change and passes after it.
Resolves: https://gitlab.com/qemu-project/qemu/-/issues/4643
Signed-off-by: Zixin Wang <w1355457260@gmail.com>
Link: https://lore.kernel.org/r/CABG6VKODisrETOuH5thhimg_DtJLH+1wmARKL5OsoND5Qse_TA@mail.gmail.com
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
diff --git a/target/i386/tcg/decode-new.c.inc b/target/i386/tcg/decode-new.c.inc
index 19c76a1200..bd9a94b316 100644
--- a/target/i386/tcg/decode-new.c.inc
+++ b/target/i386/tcg/decode-new.c.inc
@@ -1972,7 +1972,7 @@ static const X86OpEntry opcodes_root[256] = {
[0xBE] = X86_OP_ENTRYwr(MOV, LoBits,v, I,v),
[0xBF] = X86_OP_ENTRYwr(MOV, LoBits,v, I,v),
- [0xC8] = X86_OP_ENTRYrr(ENTER, I,w, I,b),
+ [0xC8] = X86_OP_ENTRYrr(ENTER, I_unsigned, w, I, b),
[0xC9] = X86_OP_ENTRY0(LEAVE),
[0xCA] = X86_OP_ENTRYr(RETF, I,w),
[0xCB] = X86_OP_ENTRY0(RETF),
diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c
index d4ccddc33f..719771032f 100644
--- a/target/i386/tcg/translate.c
+++ b/target/i386/tcg/translate.c
@@ -1989,7 +1989,7 @@ static void gen_popa(DisasContext *s)
gen_stack_update(s, 8 * size);
}
-static void gen_enter(DisasContext *s, int esp_addend, int level)
+static void gen_enter(DisasContext *s, uint16_t esp_addend, int level)
{
MemOp d_ot = mo_pushpop(s, s->dflag);
MemOp a_ot = mo_stacksize(s);
diff --git a/tests/tcg/i386/meson.build b/tests/tcg/i386/meson.build
index c912f37fab..d2ba841041 100644
--- a/tests/tcg/i386/meson.build
+++ b/tests/tcg/i386/meson.build
@@ -37,6 +37,7 @@ tests += {
'-lm', '-fno-pie', '-Wl,--no-warn-rwx-segments'],
},
'test-idiv-overflow.c': {},
+ 'test-i386-enter.c': {'cflags': ['-O2', '-fomit-frame-pointer']},
'../multiarch/sha512.c': {
'exe_name': 'sha512-sse',
'cflags': ['-msse4.1', '-O3'],
diff --git a/tests/tcg/i386/test-i386-enter.c b/tests/tcg/i386/test-i386-enter.c
new file mode 100644
index 0000000000..509740d88c
--- /dev/null
+++ b/tests/tcg/i386/test-i386-enter.c
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * ENTER's 16-bit size operand is unsigned. If the decoder sign-extends
+ * it, the stack pointer moves in the wrong direction.
+ */
+#include <stdio.h>
+
+static int check_enter(void)
+{
+ unsigned long before, after;
+
+#ifdef __x86_64__
+ __asm__ volatile(
+ "movq %%rsp, %0\n\t"
+ "enter $0x8000, $0\n\t"
+ "movq %%rsp, %1\n\t"
+ "leave\n\t"
+ : "=r"(before), "=r"(after)
+ :
+ : "memory");
+ return (before - after) == 0x8008 ? 0 : 1;
+#else
+ __asm__ volatile(
+ "movl %%esp, %0\n\t"
+ "enter $0x8000, $0\n\t"
+ "movl %%esp, %1\n\t"
+ "leave\n\t"
+ : "=r"(before), "=r"(after)
+ :
+ : "memory");
+ return (before - after) == 0x8004 ? 0 : 1;
+#endif
+}
+
+int main(void)
+{
+ if (check_enter()) {
+ printf("FAIL: ENTER treated the 16-bit size operand as signed\n");
+ return 1;
+ }
+ return 0;
+}
diff --git a/tests/tcg/x86_64/meson.build b/tests/tcg/x86_64/meson.build
index 9738d04220..1efb35cf2e 100644
--- a/tests/tcg/x86_64/meson.build
+++ b/tests/tcg/x86_64/meson.build
@@ -50,6 +50,7 @@ tests += {
'../i386/test-i386-blendv.c': {},
'../i386/test-i386-bmi2.c': {'cflags': ['-O2', '-fwrapv']},
'../i386/test-idiv-overflow.c': {},
+ '../i386/test-i386-enter.c': {'cflags': ['-O2', '-fomit-frame-pointer']},
}
tests += {