diff --git a/arch/arm64/core/offsets/offsets.c b/arch/arm64/core/offsets/offsets.c index d4e5139b4138..cf7c40aae0a3 100644 --- a/arch/arm64/core/offsets/offsets.c +++ b/arch/arm64/core/offsets/offsets.c @@ -30,16 +30,13 @@ #include #include -#ifdef CONFIG_USERSPACE -GEN_OFFSET_SYM(_thread_arch_t, priv_stack_start); -#endif - GEN_NAMED_OFFSET_SYM(_callee_saved_t, x19, x19_x20); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x21, x21_x22); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x23, x23_x24); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x25, x25_x26); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x27, x27_x28); -GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp); +GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp_el0); +GEN_NAMED_OFFSET_SYM(_callee_saved_t, sp_elx, sp_elx); GEN_ABSOLUTE_SYM(___callee_saved_t_SIZEOF, sizeof(struct _callee_saved)); diff --git a/arch/arm64/core/reset.S b/arch/arm64/core/reset.S index a29e19314a81..b2bcf78b6bfa 100644 --- a/arch/arm64/core/reset.S +++ b/arch/arm64/core/reset.S @@ -49,6 +49,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c) /* Custom plat prep_c init */ bl z_arm64_el3_plat_prep_c + /* Set SP_EL1 */ + msr sp_el1, x24 + b out 2: /* Disable alignment fault checking */ @@ -59,6 +62,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c) /* Custom plat prep_c init */ bl z_arm64_el2_plat_prep_c + /* Set SP_EL1 */ + msr sp_el1, x24 + b out 1: /* Disable alignment fault checking */ @@ -69,6 +75,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c) /* Custom plat prep_c init */ bl z_arm64_el1_plat_prep_c + /* Set SP_EL1. We cannot use sp_el1 at EL1 */ + msr SPSel, #1 + mov sp, x24 out: isb diff --git a/arch/arm64/core/switch.S b/arch/arm64/core/switch.S index 8c695595c725..6aa9ef13efd8 100644 --- a/arch/arm64/core/switch.S +++ b/arch/arm64/core/switch.S @@ -35,15 +35,19 @@ SECTION_FUNC(TEXT, z_arm64_context_switch) /* addr of callee-saved regs in thread in x2 */ add x2, x1, x3 - /* Save the current SP */ - mov x4, sp + /* Save the current SP_EL0 */ + mrs x4, sp_el0 stp x19, x20, [x2, ___callee_saved_t_x19_x20_OFFSET] stp x21, x22, [x2, ___callee_saved_t_x21_x22_OFFSET] stp x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET] stp x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET] stp x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET] - stp x29, x4, [x2, ___callee_saved_t_x29_sp_OFFSET] + stp x29, x4, [x2, ___callee_saved_t_x29_sp_el0_OFFSET] + + /* Save the current SP_ELx */ + mov x4, sp + stp x4, xzr, [x2, ___callee_saved_t_sp_elx_OFFSET] #ifdef CONFIG_SMP /* save old thread into switch handle which is required by @@ -72,8 +76,13 @@ SECTION_FUNC(TEXT, z_arm64_context_switch) ldp x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET] ldp x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET] ldp x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET] - ldp x29, x1, [x2, ___callee_saved_t_x29_sp_OFFSET] + ldp x29, x1, [x2, ___callee_saved_t_x29_sp_el0_OFFSET] + + /* Restore SP_EL0 */ + msr sp_el0, x1 + /* Restore SP_EL1 */ + ldp x1, xzr, [x2, ___callee_saved_t_sp_elx_OFFSET] mov sp, x1 #ifdef CONFIG_USERSPACE diff --git a/arch/arm64/core/thread.c b/arch/arm64/core/thread.c index af0edadfbb32..09916c81ffc0 100644 --- a/arch/arm64/core/thread.c +++ b/arch/arm64/core/thread.c @@ -16,6 +16,49 @@ #include #include +/* + * Note about stack usage: + * + * [ see also comments in include/arch/arm64/thread_stack.h ] + * + * - kernel threads are running in EL1 using SP_EL1 as stack pointer during + * normal execution and during exceptions. They are by definition already + * running in a privileged stack that is their own. + * + * - user threads are running in EL0 using SP_EL0 as stack pointer during + * normal execution. When at exception is taken or a syscall is called the + * stack pointer switches to SP_EL1 and the execution starts using the + * privileged portion of the user stack without touching SP_EL0. This portion + * is marked as not user accessible in the MMU. + * + * Kernel threads: + * + * +---------------+ <- stack_ptr + * E | ESF | + * L |<<<<<<<<<<<<<<<| <- SP_EL1 + * 1 | | + * +---------------+ + + * + * User threads: + * + * +---------------+ <- stack_ptr + * E | | + * L |<<<<<<<<<<<<<<<| <- SP_EL0 + * 0 | | + * +---------------+ ..............| + * E | ESF | | Privileged portion of the stack + * L +>>>>>>>>>>>>>>>+ <- SP_EL1 |_ used during exceptions and syscalls + * 1 | | | of size ARCH_THREAD_STACK_RESERVED + * +---------------+ <- stack_obj..| + * + * When a new user thread is created or when a kernel thread switches to user + * mode the initial ESF is relocated to the privileged portion of the stack + * and the values of stack_ptr, SP_EL0 and SP_EL1 are correctly reset when + * going through arch_user_mode_enter() and z_arm64_userspace_enter() + * + */ + #ifdef CONFIG_USERSPACE static bool is_user(struct k_thread *thread) { @@ -29,6 +72,13 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, { z_arch_esf_t *pInitCtx; + /* + * The ESF is now hosted at the top of the stack. For user threads this + * is also fine because at this stage they are still running in EL1. + * The context will be relocated by arch_user_mode_enter() before + * dropping into EL0. + */ + pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr); pInitCtx->x0 = (uint64_t)entry; @@ -52,18 +102,19 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, } else { pInitCtx->elr = (uint64_t)z_thread_entry; } - - thread->arch.priv_stack_start = 0; #else pInitCtx->elr = (uint64_t)z_thread_entry; #endif - pInitCtx->spsr = SPSR_MODE_EL1T | DAIF_FIQ_BIT; + /* Keep using SP_EL1 */ + pInitCtx->spsr = SPSR_MODE_EL1H | DAIF_FIQ_BIT; /* - * We are saving SP to pop out entry and parameters when going through - * z_arm64_exit_exc() + * We are saving SP_EL1 to pop out entry and parameters when going + * through z_arm64_exit_exc(). For user threads the definitive location + * of SP_EL1 will be set implicitly when going through + * z_arm64_userspace_enter() (see comments there) */ - thread->callee_saved.sp = (uint64_t)pInitCtx; + thread->callee_saved.sp_elx = (uint64_t)pInitCtx; thread->switch_handle = thread; } @@ -85,20 +136,24 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, /* Map the thread stack */ z_arm64_thread_pt_init(_current); - /* Setup the private stack */ - _current->arch.priv_stack_start = (uint64_t)(_current->stack_obj); - - /* Reset the stack pointer to the base discarding any old context */ + /* + * Reset the SP_EL0 stack pointer to the stack top discarding any old + * context. The actual register is written in z_arm64_userspace_enter() + */ stack_ptr = Z_STACK_PTR_ALIGN(_current->stack_info.start + _current->stack_info.size - _current->stack_info.delta); /* * Reconstruct the ESF from scratch to leverage the z_arm64_exit_exc() - * macro that will simulate a return from exception to move from EL1t - * to EL0t. On return we will be in userspace. + * macro that will simulate a return from exception to move from EL1h + * to EL0t. On return we will be in userspace using SP_EL0. + * + * We relocate the ESF to the beginning of the privileged stack in the + * not user accessible part of the stack */ - pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr); + pInitCtx = (struct __esf *) (_current->stack_obj + ARCH_THREAD_STACK_RESERVED - + sizeof(struct __esf)); pInitCtx->spsr = DAIF_FIQ_BIT | SPSR_MODE_EL0T; pInitCtx->elr = (uint64_t)z_thread_entry; @@ -109,7 +164,7 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, pInitCtx->x3 = (uint64_t)p3; /* All the needed information is already in the ESF */ - z_arm64_userspace_enter(pInitCtx); + z_arm64_userspace_enter(pInitCtx, stack_ptr); CODE_UNREACHABLE; } diff --git a/arch/arm64/core/userspace.S b/arch/arm64/core/userspace.S index 599ede99d5a3..1dbe381b03f7 100644 --- a/arch/arm64/core/userspace.S +++ b/arch/arm64/core/userspace.S @@ -115,26 +115,11 @@ valid_syscall_id: ldr x9, =_k_syscall_table ldr x9, [x9, x8, lsl #3] - /* Recover the privileged stack */ - get_cpu x10 - ldr x10, [x10, #___cpu_t_current_OFFSET] - ldr x10, [x10, #_thread_offset_to_priv_stack_start] - add x10, x10, #CONFIG_PRIVILEGED_STACK_SIZE - - /* Save the original SP on the privileged stack */ - mov x11, sp - mov sp, x10 - str x11, [sp, #-16]! - /* Jump into the syscall */ msr daifclr, #(DAIFSET_IRQ_BIT) blr x9 msr daifset, #(DAIFSET_IRQ_BIT) - /* Restore the original SP containing the ESF */ - ldr x11, [sp], #16 - mov sp, x11 - /* Save the return value into the ESF */ str x0, [sp, ___esf_t_x0_x1_OFFSET] @@ -151,5 +136,18 @@ valid_syscall_id: GTEXT(z_arm64_userspace_enter) SECTION_FUNC(TEXT, z_arm64_userspace_enter) + /* + * When a kernel thread is moved to user mode it doesn't have any + * SP_EL0 set yet. We set it here for the first time pointing to the + * beginning of the user accessible part of the stack (the top). + */ + msr sp_el0, x1 + + /* + * Set SP_EL1 to point at the end of the ESF. Since we have relocated + * the ESF at the beginning of the privileged stack area, when the ESF + * is popped out by z_arm64_exit_exc() the SP_EL1 will be at the right + * location for when the next exception will come. + */ mov sp, x0 b z_arm64_exit_exc diff --git a/arch/arm64/core/vector_table.S b/arch/arm64/core/vector_table.S index 29c85343ba7e..78c940b73f87 100644 --- a/arch/arm64/core/vector_table.S +++ b/arch/arm64/core/vector_table.S @@ -25,10 +25,6 @@ _ASM_FILE_PROLOGUE */ .macro z_arm64_enter_exc xreg0, xreg1 - - /* Switch to SP_EL0 */ - msr spsel, #0 - /* * Two things can happen to the remaining registers: * diff --git a/arch/arm64/include/kernel_arch_func.h b/arch/arm64/include/kernel_arch_func.h index 9a8ed87c1044..09bd93724237 100644 --- a/arch/arm64/include/kernel_arch_func.h +++ b/arch/arm64/include/kernel_arch_func.h @@ -40,7 +40,7 @@ static inline void arch_switch(void *switch_to, void **switched_from) } extern void z_arm64_fatal_error(z_arch_esf_t *esf, unsigned int reason); -extern void z_arm64_userspace_enter(z_arch_esf_t *esf); +extern void z_arm64_userspace_enter(z_arch_esf_t *esf, uintptr_t sp_el0); extern void z_arm64_set_ttbr0(uintptr_t ttbr0); extern void z_arm64_ptable_ipi(void); diff --git a/arch/arm64/include/offsets_short_arch.h b/arch/arm64/include/offsets_short_arch.h index f8b49a4cff08..7df7bf004d23 100644 --- a/arch/arm64/include/offsets_short_arch.h +++ b/arch/arm64/include/offsets_short_arch.h @@ -9,9 +9,4 @@ #include -#ifdef CONFIG_USERSPACE -#define _thread_offset_to_priv_stack_start \ - (___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_start_OFFSET) -#endif - #endif /* ZEPHYR_ARCH_ARM64_INCLUDE_OFFSETS_SHORT_ARCH_H_ */ diff --git a/include/arch/arm64/thread.h b/include/arch/arm64/thread.h index 44d6e7b5f31e..6b20665f70de 100644 --- a/include/arch/arm64/thread.h +++ b/include/arch/arm64/thread.h @@ -33,8 +33,10 @@ struct _callee_saved { uint64_t x26; uint64_t x27; uint64_t x28; - uint64_t x29; /* FP */ - uint64_t sp; + uint64_t x29; + uint64_t sp_el0; + uint64_t sp_elx; + uint64_t xzr; }; typedef struct _callee_saved _callee_saved_t; @@ -42,7 +44,6 @@ typedef struct _callee_saved _callee_saved_t; struct _thread_arch { #ifdef CONFIG_USERSPACE struct arm_mmu_ptables *ptables; - uint64_t priv_stack_start; #endif }; diff --git a/include/arch/arm64/thread_stack.h b/include/arch/arm64/thread_stack.h index 317a5c20e15c..422a0dcf9665 100644 --- a/include/arch/arm64/thread_stack.h +++ b/include/arch/arm64/thread_stack.h @@ -18,6 +18,8 @@ #endif /* + * [ see also comments in arch/arm64/core/thread.c ] + * * High memory addresses * * +-------------------+ <- thread.stack_info.start + thread.stack_info.size @@ -31,7 +33,7 @@ * | Unused stack | * | | * +-------------------+ <- thread.stack_info.start - * | Reserved memory | } K_(THREAD|KERNEL)_STACK_RESERVED + * | Privileged stack | } K_(THREAD|KERNEL)_STACK_RESERVED * +-------------------+ <- thread.stack_obj * * Low Memory addresses diff --git a/subsys/debug/thread_info.c b/subsys/debug/thread_info.c index 09cb37319ad3..74bd4b19ea69 100644 --- a/subsys/debug/thread_info.c +++ b/subsys/debug/thread_info.c @@ -51,8 +51,9 @@ size_t _kernel_thread_info_offsets[] = { user_options), [THREAD_INFO_OFFSET_T_PRIO] = offsetof(struct _thread_base, prio), #if defined(CONFIG_ARM64) + /* We are assuming that the SP of interest is SP_EL1 */ [THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread, - callee_saved.sp), + callee_saved.sp_elx), #elif defined(CONFIG_ARM) [THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread, callee_saved.psp),