From 747ffc2fcf969eff9309d7f2d1d61cb8b9e1bb40 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 3 May 2020 13:03:54 +0100 Subject: [PATCH 1/3] ARM: uaccess: consolidate uaccess asm to asm/uaccess-asm.h Consolidate the user access assembly code to asm/uaccess-asm.h. This moves the csdb, check_uaccess, uaccess_mask_range_ptr, uaccess_enable, uaccess_disable, uaccess_save, uaccess_restore macros, and creates two new ones for exception entry and exit - uaccess_entry and uaccess_exit. This makes the uaccess_save and uaccess_restore macros private to asm/uaccess-asm.h. Signed-off-by: Russell King --- arch/arm/include/asm/assembler.h | 75 +------------------- arch/arm/include/asm/uaccess-asm.h | 106 +++++++++++++++++++++++++++++ arch/arm/kernel/entry-armv.S | 11 +-- arch/arm/kernel/entry-header.S | 9 +-- 4 files changed, 112 insertions(+), 89 deletions(-) create mode 100644 arch/arm/include/asm/uaccess-asm.h diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 99929122dad7..3546d294d55f 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -18,11 +18,11 @@ #endif #include -#include #include #include #include #include +#include #define IOMEM(x) (x) @@ -446,79 +446,6 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .size \name , . - \name .endm - .macro csdb -#ifdef CONFIG_THUMB2_KERNEL - .inst.w 0xf3af8014 -#else - .inst 0xe320f014 -#endif - .endm - - .macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req -#ifndef CONFIG_CPU_USE_DOMAINS - adds \tmp, \addr, #\size - 1 - sbcscc \tmp, \tmp, \limit - bcs \bad -#ifdef CONFIG_CPU_SPECTRE - movcs \addr, #0 - csdb -#endif -#endif - .endm - - .macro uaccess_mask_range_ptr, addr:req, size:req, limit:req, tmp:req -#ifdef CONFIG_CPU_SPECTRE - sub \tmp, \limit, #1 - subs \tmp, \tmp, \addr @ tmp = limit - 1 - addr - addhs \tmp, \tmp, #1 @ if (tmp >= 0) { - subshs \tmp, \tmp, \size @ tmp = limit - (addr + size) } - movlo \addr, #0 @ if (tmp < 0) addr = NULL - csdb -#endif - .endm - - .macro uaccess_disable, tmp, isb=1 -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - /* - * Whenever we re-enter userspace, the domains should always be - * set appropriately. - */ - mov \tmp, #DACR_UACCESS_DISABLE - mcr p15, 0, \tmp, c3, c0, 0 @ Set domain register - .if \isb - instr_sync - .endif -#endif - .endm - - .macro uaccess_enable, tmp, isb=1 -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - /* - * Whenever we re-enter userspace, the domains should always be - * set appropriately. - */ - mov \tmp, #DACR_UACCESS_ENABLE - mcr p15, 0, \tmp, c3, c0, 0 - .if \isb - instr_sync - .endif -#endif - .endm - - .macro uaccess_save, tmp -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - mrc p15, 0, \tmp, c3, c0, 0 - str \tmp, [sp, #SVC_DACR] -#endif - .endm - - .macro uaccess_restore -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - ldr r0, [sp, #SVC_DACR] - mcr p15, 0, r0, c3, c0, 0 -#endif - .endm - .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo .macro ret\c, reg #if __LINUX_ARM_ARCH__ < 6 diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h new file mode 100644 index 000000000000..d475e3e8145d --- /dev/null +++ b/arch/arm/include/asm/uaccess-asm.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_UACCESS_ASM_H__ +#define __ASM_UACCESS_ASM_H__ + +#include +#include +#include +#include + + .macro csdb +#ifdef CONFIG_THUMB2_KERNEL + .inst.w 0xf3af8014 +#else + .inst 0xe320f014 +#endif + .endm + + .macro check_uaccess, addr:req, size:req, limit:req, tmp:req, bad:req +#ifndef CONFIG_CPU_USE_DOMAINS + adds \tmp, \addr, #\size - 1 + sbcscc \tmp, \tmp, \limit + bcs \bad +#ifdef CONFIG_CPU_SPECTRE + movcs \addr, #0 + csdb +#endif +#endif + .endm + + .macro uaccess_mask_range_ptr, addr:req, size:req, limit:req, tmp:req +#ifdef CONFIG_CPU_SPECTRE + sub \tmp, \limit, #1 + subs \tmp, \tmp, \addr @ tmp = limit - 1 - addr + addhs \tmp, \tmp, #1 @ if (tmp >= 0) { + subshs \tmp, \tmp, \size @ tmp = limit - (addr + size) } + movlo \addr, #0 @ if (tmp < 0) addr = NULL + csdb +#endif + .endm + + .macro uaccess_disable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_DISABLE + mcr p15, 0, \tmp, c3, c0, 0 @ Set domain register + .if \isb + instr_sync + .endif +#endif + .endm + + .macro uaccess_enable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_ENABLE + mcr p15, 0, \tmp, c3, c0, 0 + .if \isb + instr_sync + .endif +#endif + .endm + + .macro uaccess_save, tmp +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + mrc p15, 0, \tmp, c3, c0, 0 + str \tmp, [sp, #SVC_DACR] +#endif + .endm + + .macro uaccess_restore +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + ldr r0, [sp, #SVC_DACR] + mcr p15, 0, r0, c3, c0, 0 +#endif + .endm + + /* + * Save the address limit on entry to a privileged exception and + * if using PAN, save and disable usermode access. + */ + .macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable + ldr \tmp0, [\tsk, #TI_ADDR_LIMIT] + mov \tmp1, #TASK_SIZE + str \tmp1, [\tsk, #TI_ADDR_LIMIT] + str \tmp0, [sp, #SVC_ADDR_LIMIT] + uaccess_save \tmp0 + .if \disable + uaccess_disable \tmp0 + .endif + .endm + + /* Restore the user access state previously saved by uaccess_entry */ + .macro uaccess_exit, tsk, tmp0, tmp1 + ldr \tmp1, [sp, #SVC_ADDR_LIMIT] + uaccess_restore + str \tmp1, [\tsk, #TI_ADDR_LIMIT] + .endm + +#endif /* __ASM_UACCESS_ASM_H__ */ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 77f54830554c..55a47df04773 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -27,6 +27,7 @@ #include #include #include +#include #include "entry-header.S" #include @@ -179,15 +180,7 @@ ENDPROC(__und_invalid) stmia r7, {r2 - r6} get_thread_info tsk - ldr r0, [tsk, #TI_ADDR_LIMIT] - mov r1, #TASK_SIZE - str r1, [tsk, #TI_ADDR_LIMIT] - str r0, [sp, #SVC_ADDR_LIMIT] - - uaccess_save r0 - .if \uaccess - uaccess_disable r0 - .endif + uaccess_entry tsk, r0, r1, r2, \uaccess .if \trace #ifdef CONFIG_TRACE_IRQFLAGS diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 32051ec5b33f..40db0f9188b6 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -6,6 +6,7 @@ #include #include #include +#include #include @ Bad Abort numbers @@ -217,9 +218,7 @@ blne trace_hardirqs_off #endif .endif - ldr r1, [sp, #SVC_ADDR_LIMIT] - uaccess_restore - str r1, [tsk, #TI_ADDR_LIMIT] + uaccess_exit tsk, r0, r1 #ifndef CONFIG_THUMB2_KERNEL @ ARM mode SVC restore @@ -263,9 +262,7 @@ @ on the stack remains correct). @ .macro svc_exit_via_fiq - ldr r1, [sp, #SVC_ADDR_LIMIT] - uaccess_restore - str r1, [tsk, #TI_ADDR_LIMIT] + uaccess_exit tsk, r0, r1 #ifndef CONFIG_THUMB2_KERNEL @ ARM mode restore mov r0, sp From 8ede890b0bcebe8c760aacfe20e934d98c3dc6aa Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 3 May 2020 13:14:09 +0100 Subject: [PATCH 2/3] ARM: uaccess: integrate uaccess_save and uaccess_restore Integrate uaccess_save / uaccess_restore macros into the new uaccess_entry / uaccess_exit macros respectively. Signed-off-by: Russell King --- arch/arm/include/asm/uaccess-asm.h | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h index d475e3e8145d..e46468b91eaa 100644 --- a/arch/arm/include/asm/uaccess-asm.h +++ b/arch/arm/include/asm/uaccess-asm.h @@ -67,30 +67,23 @@ #endif .endm - .macro uaccess_save, tmp #ifdef CONFIG_CPU_SW_DOMAIN_PAN - mrc p15, 0, \tmp, c3, c0, 0 - str \tmp, [sp, #SVC_DACR] +#define DACR(x...) x +#else +#define DACR(x...) #endif - .endm - - .macro uaccess_restore -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - ldr r0, [sp, #SVC_DACR] - mcr p15, 0, r0, c3, c0, 0 -#endif - .endm /* * Save the address limit on entry to a privileged exception and * if using PAN, save and disable usermode access. */ .macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable - ldr \tmp0, [\tsk, #TI_ADDR_LIMIT] - mov \tmp1, #TASK_SIZE - str \tmp1, [\tsk, #TI_ADDR_LIMIT] - str \tmp0, [sp, #SVC_ADDR_LIMIT] - uaccess_save \tmp0 + ldr \tmp1, [\tsk, #TI_ADDR_LIMIT] + mov \tmp2, #TASK_SIZE + str \tmp2, [\tsk, #TI_ADDR_LIMIT] + DACR( mrc p15, 0, \tmp0, c3, c0, 0) + DACR( str \tmp0, [sp, #SVC_DACR]) + str \tmp1, [sp, #SVC_ADDR_LIMIT] .if \disable uaccess_disable \tmp0 .endif @@ -99,8 +92,11 @@ /* Restore the user access state previously saved by uaccess_entry */ .macro uaccess_exit, tsk, tmp0, tmp1 ldr \tmp1, [sp, #SVC_ADDR_LIMIT] - uaccess_restore + DACR( ldr \tmp0, [sp, #SVC_DACR]) str \tmp1, [\tsk, #TI_ADDR_LIMIT] + DACR( mcr p15, 0, \tmp0, c3, c0, 0) .endm +#undef DACR + #endif /* __ASM_UACCESS_ASM_H__ */ From 71f8af1110101facfad68989ff91f88f8e2c3e22 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 3 May 2020 13:24:07 +0100 Subject: [PATCH 3/3] ARM: uaccess: fix DACR mismatch with nested exceptions Tomas Paukrt reports that his SAM9X60 based system (ARM926, ARMv5TJ) fails to fix up alignment faults, eventually resulting in a kernel oops. The problem occurs when using CONFIG_CPU_USE_DOMAINS with commit e6978e4bf181 ("ARM: save and reset the address limit when entering an exception"). This is because the address limit is set back to TASK_SIZE on exception entry, and, although it is restored on exception exit, the domain register is not. Hence, this sequence can occur: interrupt pt_regs->addr_limit = addr_limit // USER_DS addr_limit = USER_DS alignment exception __probe_kernel_read() old_fs = get_fs() // USER_DS set_fs(KERNEL_DS) addr_limit = KERNEL_DS dacr.kernel = DOMAIN_MANAGER interrupt pt_regs->addr_limit = addr_limit // KERNEL_DS addr_limit = USER_DS alignment exception __probe_kernel_read() old_fs = get_fs() // USER_DS set_fs(KERNEL_DS) addr_limit = KERNEL_DS dacr.kernel = DOMAIN_MANAGER ... set_fs(old_fs) addr_limit = USER_DS dacr.kernel = DOMAIN_CLIENT ... addr_limit = pt_regs->addr_limit // KERNEL_DS interrupt returns At this point, addr_limit is correctly restored to KERNEL_DS for __probe_kernel_read() to continue execution, but dacr.kernel is not, it has been reset by the set_fs(old_fs) to DOMAIN_CLIENT. This would not have happened prior to the mentioned commit, because addr_limit would remain KERNEL_DS, so get_fs() would have returned KERNEL_DS, and so would correctly nest. This commit fixes the problem by also saving the DACR on exception entry if either CONFIG_CPU_SW_DOMAIN_PAN or CONFIG_CPU_USE_DOMAINS are enabled, and resetting the DACR appropriately on exception entry to match addr_limit and PAN settings. Fixes: e6978e4bf181 ("ARM: save and reset the address limit when entering an exception") Reported-by: Tomas Paukrt Signed-off-by: Russell King --- arch/arm/include/asm/uaccess-asm.h | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h index e46468b91eaa..907571fd05c6 100644 --- a/arch/arm/include/asm/uaccess-asm.h +++ b/arch/arm/include/asm/uaccess-asm.h @@ -67,15 +67,21 @@ #endif .endm -#ifdef CONFIG_CPU_SW_DOMAIN_PAN +#if defined(CONFIG_CPU_SW_DOMAIN_PAN) || defined(CONFIG_CPU_USE_DOMAINS) #define DACR(x...) x #else #define DACR(x...) #endif /* - * Save the address limit on entry to a privileged exception and - * if using PAN, save and disable usermode access. + * Save the address limit on entry to a privileged exception. + * + * If we are using the DACR for kernel access by the user accessors + * (CONFIG_CPU_USE_DOMAINS=y), always reset the DACR kernel domain + * back to client mode, whether or not \disable is set. + * + * If we are using SW PAN, set the DACR user domain to no access + * if \disable is set. */ .macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable ldr \tmp1, [\tsk, #TI_ADDR_LIMIT] @@ -84,8 +90,17 @@ DACR( mrc p15, 0, \tmp0, c3, c0, 0) DACR( str \tmp0, [sp, #SVC_DACR]) str \tmp1, [sp, #SVC_ADDR_LIMIT] - .if \disable - uaccess_disable \tmp0 + .if \disable && IS_ENABLED(CONFIG_CPU_SW_DOMAIN_PAN) + /* kernel=client, user=no access */ + mov \tmp2, #DACR_UACCESS_DISABLE + mcr p15, 0, \tmp2, c3, c0, 0 + instr_sync + .elseif IS_ENABLED(CONFIG_CPU_USE_DOMAINS) + /* kernel=client */ + bic \tmp2, \tmp0, #domain_mask(DOMAIN_KERNEL) + orr \tmp2, \tmp2, #domain_val(DOMAIN_KERNEL, DOMAIN_CLIENT) + mcr p15, 0, \tmp2, c3, c0, 0 + instr_sync .endif .endm