Commit fd2dee1c6e22 for kernel
commit fd2dee1c6e2256f726ba33fd3083a7be0efc80d3
Author: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
Date: Fri Dec 5 10:52:12 2025 +0000
ARM: fix branch predictor hardening
__do_user_fault() may be called with indeterminent interrupt enable
state, which means we may be preemptive at this point. This causes
problems when calling harden_branch_predictor(). For example, when
called from a data abort, do_alignment_fault()->do_bad_area().
Move harden_branch_predictor() out of __do_user_fault() and into the
calling contexts.
Moving it into do_kernel_address_page_fault(), we can be sure that
interrupts will be disabled here.
Converting do_translation_fault() to use do_kernel_address_page_fault()
rather than do_bad_area() means that we keep branch predictor handling
for translation faults. Interrupts will also be disabled at this call
site.
do_sect_fault() needs special handling, so detect user mode accesses
to kernel-addresses, and add an explicit call to branch predictor
hardening.
Finally, add branch predictor hardening to do_alignment() for the
faulting case (user mode accessing kernel addresses) before interrupts
are enabled.
This should cover all cases where harden_branch_predictor() is called,
ensuring that it is always has interrupts disabled, also ensuring that
it is called early in each call path.
Reviewed-by: Xie Yuanbin <xieyuanbin1@huawei.com>
Tested-by: Xie Yuanbin <xieyuanbin1@huawei.com>
Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 3c6ddb1afdc4..812380f30ae3 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -19,10 +19,11 @@
#include <linux/init.h>
#include <linux/sched/signal.h>
#include <linux/uaccess.h>
+#include <linux/unaligned.h>
#include <asm/cp15.h>
#include <asm/system_info.h>
-#include <linux/unaligned.h>
+#include <asm/system_misc.h>
#include <asm/opcodes.h>
#include "fault.h"
@@ -809,6 +810,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
int thumb2_32b = 0;
int fault;
+ if (addr >= TASK_SIZE && user_mode(regs))
+ harden_branch_predictor();
+
if (interrupts_enabled(regs))
local_irq_enable();
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 0e5b4bc7b217..ed4330cc3f4e 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -198,9 +198,6 @@ __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
{
struct task_struct *tsk = current;
- if (addr > TASK_SIZE)
- harden_branch_predictor();
-
#ifdef CONFIG_DEBUG_USER
if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
((user_debug & UDBG_BUS) && (sig == SIGBUS))) {
@@ -269,8 +266,10 @@ do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,
/*
* Fault from user mode for a kernel space address. User mode
* should not be faulting in kernel space, which includes the
- * vector/khelper page. Send a SIGSEGV.
+ * vector/khelper page. Handle the branch predictor hardening
+ * while interrupts are still disabled, then send a SIGSEGV.
*/
+ harden_branch_predictor();
__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
} else {
/*
@@ -485,16 +484,20 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
* We enter here because the first level page table doesn't contain
* a valid entry for the address.
*
- * If the address is in kernel space (>= TASK_SIZE), then we are
- * probably faulting in the vmalloc() area.
+ * If this is a user address (addr < TASK_SIZE), we handle this as a
+ * normal page fault. This leaves the remainder of the function to handle
+ * kernel address translation faults.
*
- * If the init_task's first level page tables contains the relevant
- * entry, we copy the it to this task. If not, we send the process
- * a signal, fixup the exception, or oops the kernel.
+ * Since user mode is not permitted to access kernel addresses, pass these
+ * directly to do_kernel_address_page_fault() to handle.
*
- * NOTE! We MUST NOT take any locks for this case. We may be in an
- * interrupt or a critical region, and should only copy the information
- * from the master page table, nothing more.
+ * Otherwise, we're probably faulting in the vmalloc() area, so try to fix
+ * that up. Note that we must not take any locks or enable interrupts in
+ * this case.
+ *
+ * If vmalloc() fixup fails, that means the non-leaf page tables did not
+ * contain an entry for this address, so handle this via
+ * do_kernel_address_page_fault().
*/
#ifdef CONFIG_MMU
static int __kprobes
@@ -560,7 +563,8 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
return 0;
bad_area:
- do_bad_area(addr, fsr, regs);
+ do_kernel_address_page_fault(current->mm, addr, fsr, regs);
+
return 0;
}
#else /* CONFIG_MMU */
@@ -580,7 +584,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
static int
do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
{
+ /*
+ * If this is a kernel address, but from user mode, then userspace
+ * is trying bad stuff. Invoke the branch predictor handling.
+ * Interrupts are disabled here.
+ */
+ if (addr >= TASK_SIZE && user_mode(regs))
+ harden_branch_predictor();
+
do_bad_area(addr, fsr, regs);
+
return 0;
}
#endif /* CONFIG_ARM_LPAE */