Commit c885271538 for qemu.org
commit c885271538b47311d05e328e6ac7589c393ebffc
Author: Richard Henderson <richard.henderson@linaro.org>
Date: Mon Nov 16 16:00:38 2026 -0800
accel/tcg: Fix jump cache flush after large page flush
When a large page flush occurs, flush the entire jump cache.
Resolves: https://gitlab.com/qemu-project/qemu/-/work_items/4873
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Reviewed-by: Philippe Mathieu-Daudé <philmd@oss.qualcomm.com>
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 3ca484b039..d79eae58df 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -472,7 +472,8 @@ static inline void tlb_flush_vtlb_page_locked(CPUState *cpu, int mmu_idx,
tlb_flush_vtlb_page_mask_locked(cpu, mmu_idx, page, -1);
}
-static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
+/* Return true if the entire tlb was flushed. */
+static bool tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
{
vaddr lp_addr = cpu->neg.tlb.d[midx].large_page_addr;
vaddr lp_mask = cpu->neg.tlb.d[midx].large_page_mask;
@@ -480,12 +481,13 @@ static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
/* Check if we need to flush due to large pages. */
if ((page & lp_mask) == lp_addr) {
tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
- } else {
- if (tlb_flush_entry_locked(tlb_entry(cpu, midx, page), page)) {
- tlb_n_used_entries_dec(cpu, midx);
- }
- tlb_flush_vtlb_page_locked(cpu, midx, page);
+ return true;
}
+ if (tlb_flush_entry_locked(tlb_entry(cpu, midx, page), page)) {
+ tlb_n_used_entries_dec(cpu, midx);
+ }
+ tlb_flush_vtlb_page_locked(cpu, midx, page);
+ return false;
}
/**
@@ -502,13 +504,14 @@ static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
MMUIdxMap idxmap)
{
int mmu_idx;
+ bool flush_all = false;
assert_cpu_is_self_or_stopped(cpu);
qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if ((idxmap >> mmu_idx) & 1) {
- tlb_flush_page_locked(cpu, mmu_idx, addr);
+ flush_all |= tlb_flush_page_locked(cpu, mmu_idx, addr);
}
}
qemu_spin_unlock(&cpu->neg.tlb.c.lock);
@@ -517,8 +520,12 @@ static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
* Discard jump cache entries for any tb which might potentially
* overlap the flushed page, which includes the previous.
*/
- tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
- tb_jmp_cache_clear_page(cpu, addr);
+ if (flush_all) {
+ tcg_flush_jmp_cache(cpu);
+ } else {
+ tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
+ tb_jmp_cache_clear_page(cpu, addr);
+ }
}
/**
@@ -624,7 +631,8 @@ void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr)
tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
}
-static void tlb_flush_range_locked(CPUState *cpu, int midx,
+/* Return true if the entire tlb was flushed. */
+static bool tlb_flush_range_locked(CPUState *cpu, int midx,
vaddr addr, vaddr len,
unsigned bits)
{
@@ -644,7 +652,7 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx,
*/
if (mask < f->mask || len > f->mask) {
tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
- return;
+ return true;
}
/*
@@ -654,7 +662,7 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx,
*/
if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
- return;
+ return true;
}
for (vaddr i = 0; i < len; i += TARGET_PAGE_SIZE) {
@@ -666,6 +674,7 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx,
}
tlb_flush_vtlb_page_mask_locked(cpu, midx, page, mask);
}
+ return false;
}
typedef struct {
@@ -678,6 +687,7 @@ typedef struct {
static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
TLBFlushRangeData d)
{
+ bool flush_all = false;
int mmu_idx;
assert_cpu_is_self_or_stopped(cpu);
@@ -685,7 +695,8 @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if ((d.idxmap >> mmu_idx) & 1) {
- tlb_flush_range_locked(cpu, mmu_idx, d.addr, d.len, d.bits);
+ flush_all |= tlb_flush_range_locked(cpu, mmu_idx, d.addr,
+ d.len, d.bits);
}
}
qemu_spin_unlock(&cpu->neg.tlb.c.lock);
@@ -694,7 +705,7 @@ static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
* If the length is larger than the jump cache size, then it will take
* longer to clear each entry individually than it will to clear it all.
*/
- if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) {
+ if (flush_all || d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) {
tcg_flush_jmp_cache(cpu);
return;
}