arm64: Rework stack usage

The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.

If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.

The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.

We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.

When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.

Signed-off-by: Carlo Caione <ccaione@baylibre.com>

arch/arm64/core/offsets/offsets.c

@@ -30,16 +30,13 @@
 #include <kernel_arch_data.h>
 #include <kernel_offsets.h>
 
-#ifdef CONFIG_USERSPACE
-GEN_OFFSET_SYM(_thread_arch_t, priv_stack_start);
-#endif
-
 GEN_NAMED_OFFSET_SYM(_callee_saved_t, x19, x19_x20);
 GEN_NAMED_OFFSET_SYM(_callee_saved_t, x21, x21_x22);
 GEN_NAMED_OFFSET_SYM(_callee_saved_t, x23, x23_x24);
 GEN_NAMED_OFFSET_SYM(_callee_saved_t, x25, x25_x26);
 GEN_NAMED_OFFSET_SYM(_callee_saved_t, x27, x27_x28);
-GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp);
+GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp_el0);
+GEN_NAMED_OFFSET_SYM(_callee_saved_t, sp_elx, sp_elx);
 
 GEN_ABSOLUTE_SYM(___callee_saved_t_SIZEOF, sizeof(struct _callee_saved));
 

arch/arm64/core/reset.S

@@ -49,6 +49,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
 	/* Custom plat prep_c init */
 	bl	z_arm64_el3_plat_prep_c
 
+	/* Set SP_EL1 */
+	msr     sp_el1, x24
+
 	b	out
 2:
 	/* Disable alignment fault checking */
@@ -59,6 +62,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
 	/* Custom plat prep_c init */
 	bl	z_arm64_el2_plat_prep_c
 
+	/* Set SP_EL1 */
+	msr     sp_el1, x24
+
 	b	out
 1:
 	/* Disable alignment fault checking */
@@ -69,6 +75,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
 	/* Custom plat prep_c init */
 	bl	z_arm64_el1_plat_prep_c
 
+	/* Set SP_EL1. We cannot use sp_el1 at EL1 */
+	msr     SPSel, #1
+	mov     sp, x24
 out:
 	isb
 

arch/arm64/core/switch.S

@@ -35,15 +35,19 @@ SECTION_FUNC(TEXT, z_arm64_context_switch)
 	/* addr of callee-saved regs in thread in x2 */
 	add	x2, x1, x3
 
-	/* Save the current SP */
-	mov	x4, sp
+	/* Save the current SP_EL0 */
+	mrs	x4, sp_el0
 
 	stp	x19, x20, [x2, ___callee_saved_t_x19_x20_OFFSET]
 	stp	x21, x22, [x2, ___callee_saved_t_x21_x22_OFFSET]
 	stp	x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET]
 	stp	x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET]
 	stp	x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET]
-	stp	x29, x4, [x2, ___callee_saved_t_x29_sp_OFFSET]
+	stp	x29, x4,  [x2, ___callee_saved_t_x29_sp_el0_OFFSET]
+
+	/* Save the current SP_ELx */
+	mov	x4, sp
+	stp	x4, xzr,  [x2, ___callee_saved_t_sp_elx_OFFSET]
 
 #ifdef CONFIG_SMP
 	/* save old thread into switch handle which is required by
@@ -72,8 +76,13 @@ SECTION_FUNC(TEXT, z_arm64_context_switch)
 	ldp	x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET]
 	ldp	x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET]
 	ldp	x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET]
-	ldp	x29, x1, [x2, ___callee_saved_t_x29_sp_OFFSET]
+	ldp	x29, x1,  [x2, ___callee_saved_t_x29_sp_el0_OFFSET]
+
+	/* Restore SP_EL0 */
+	msr	sp_el0, x1
 
+	/* Restore SP_EL1 */
+	ldp	x1, xzr, [x2, ___callee_saved_t_sp_elx_OFFSET]
 	mov	sp, x1
 
 #ifdef CONFIG_USERSPACE

arch/arm64/core/thread.c

@@ -16,6 +16,49 @@
 #include <wait_q.h>
 #include <arch/cpu.h>
 
+/*
+ * Note about stack usage:
+ *
+ * [ see also comments in include/arch/arm64/thread_stack.h ]
+ *
+ * - kernel threads are running in EL1 using SP_EL1 as stack pointer during
+ *   normal execution and during exceptions. They are by definition already
+ *   running in a privileged stack that is their own.
+ *
+ * - user threads are running in EL0 using SP_EL0 as stack pointer during
+ *   normal execution. When at exception is taken or a syscall is called the
+ *   stack pointer switches to SP_EL1 and the execution starts using the
+ *   privileged portion of the user stack without touching SP_EL0. This portion
+ *   is marked as not user accessible in the MMU.
+ *
+ *   Kernel threads:
+ *
+ *    +---------------+ <- stack_ptr
+ *  E |     ESF       |
+ *  L |<<<<<<<<<<<<<<<| <- SP_EL1
+ *  1 |               |
+ *    +---------------+
+
+ *
+ *   User threads:
+ *
+ *    +---------------+ <- stack_ptr
+ *  E |               |
+ *  L |<<<<<<<<<<<<<<<| <- SP_EL0
+ *  0 |               |
+ *    +---------------+ ..............|
+ *  E |     ESF       |               |  Privileged portion of the stack
+ *  L +>>>>>>>>>>>>>>>+ <- SP_EL1     |_ used during exceptions and syscalls
+ *  1 |               |               |  of size ARCH_THREAD_STACK_RESERVED
+ *    +---------------+ <- stack_obj..|
+ *
+ *  When a new user thread is created or when a kernel thread switches to user
+ *  mode the initial ESF is relocated to the privileged portion of the stack
+ *  and the values of stack_ptr, SP_EL0 and SP_EL1 are correctly reset when
+ *  going through arch_user_mode_enter() and z_arm64_userspace_enter()
+ *
+ */
+
 #ifdef CONFIG_USERSPACE
 static bool is_user(struct k_thread *thread)
 {
@@ -29,6 +72,13 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
 {
 	z_arch_esf_t *pInitCtx;
 
+	/*
+	 * The ESF is now hosted at the top of the stack. For user threads this
+	 * is also fine because at this stage they are still running in EL1.
+	 * The context will be relocated by arch_user_mode_enter() before
+	 * dropping into EL0.
+	 */
+
 	pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr);
 
 	pInitCtx->x0 = (uint64_t)entry;
@@ -52,18 +102,19 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
 	} else {
 		pInitCtx->elr = (uint64_t)z_thread_entry;
 	}
-
-	thread->arch.priv_stack_start = 0;
 #else
 	pInitCtx->elr = (uint64_t)z_thread_entry;
 #endif
-	pInitCtx->spsr = SPSR_MODE_EL1T | DAIF_FIQ_BIT;
+	/* Keep using SP_EL1 */
+	pInitCtx->spsr = SPSR_MODE_EL1H | DAIF_FIQ_BIT;
 
 	/*
-	 * We are saving SP to pop out entry and parameters when going through
-	 * z_arm64_exit_exc()
+	 * We are saving SP_EL1 to pop out entry and parameters when going
+	 * through z_arm64_exit_exc(). For user threads the definitive location
+	 * of SP_EL1 will be set implicitly when going through
+	 * z_arm64_userspace_enter() (see comments there)
 	 */
-	thread->callee_saved.sp = (uint64_t)pInitCtx;
+	thread->callee_saved.sp_elx = (uint64_t)pInitCtx;
 
 	thread->switch_handle = thread;
 }
@@ -85,20 +136,24 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
 	/* Map the thread stack */
 	z_arm64_thread_pt_init(_current);
 
-	/* Setup the private stack */
-	_current->arch.priv_stack_start = (uint64_t)(_current->stack_obj);
-
-	/* Reset the stack pointer to the base discarding any old context */
+	/*
+	 * Reset the SP_EL0 stack pointer to the stack top discarding any old
+	 * context. The actual register is written in z_arm64_userspace_enter()
+	 */
 	stack_ptr = Z_STACK_PTR_ALIGN(_current->stack_info.start +
 				      _current->stack_info.size -
 				      _current->stack_info.delta);
 
 	/*
 	 * Reconstruct the ESF from scratch to leverage the z_arm64_exit_exc()
-	 * macro that will simulate a return from exception to move from EL1t
-	 * to EL0t. On return we will be in userspace.
+	 * macro that will simulate a return from exception to move from EL1h
+	 * to EL0t. On return we will be in userspace using SP_EL0.
+	 *
+	 * We relocate the ESF to the beginning of the privileged stack in the
+	 * not user accessible part of the stack
 	 */
-	pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr);
+	pInitCtx = (struct __esf *) (_current->stack_obj + ARCH_THREAD_STACK_RESERVED -
+			sizeof(struct __esf));
 
 	pInitCtx->spsr = DAIF_FIQ_BIT | SPSR_MODE_EL0T;
 	pInitCtx->elr = (uint64_t)z_thread_entry;
@@ -109,7 +164,7 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
 	pInitCtx->x3 = (uint64_t)p3;
 
 	/* All the needed information is already in the ESF */
-	z_arm64_userspace_enter(pInitCtx);
+	z_arm64_userspace_enter(pInitCtx, stack_ptr);
 
 	CODE_UNREACHABLE;
 }

arch/arm64/core/userspace.S

@@ -115,26 +115,11 @@ valid_syscall_id:
 	ldr	x9, =_k_syscall_table
 	ldr	x9, [x9, x8, lsl #3]
 
-	/* Recover the privileged stack */
-	get_cpu	x10
-	ldr	x10, [x10, #___cpu_t_current_OFFSET]
-	ldr     x10, [x10, #_thread_offset_to_priv_stack_start]
-	add	x10, x10, #CONFIG_PRIVILEGED_STACK_SIZE
-
-	/* Save the original SP on the privileged stack */
-	mov	x11, sp
-	mov	sp, x10
-	str	x11,  [sp, #-16]!
-
 	/* Jump into the syscall */
 	msr	daifclr, #(DAIFSET_IRQ_BIT)
 	blr	x9
 	msr	daifset, #(DAIFSET_IRQ_BIT)
 
-	/* Restore the original SP containing the ESF */
-	ldr	x11, [sp], #16
-	mov	sp, x11
-
 	/* Save the return value into the ESF */
 	str	x0, [sp, ___esf_t_x0_x1_OFFSET]
 
@@ -151,5 +136,18 @@ valid_syscall_id:
 
 GTEXT(z_arm64_userspace_enter)
 SECTION_FUNC(TEXT, z_arm64_userspace_enter)
+	/*
+	 * When a kernel thread is moved to user mode it doesn't have any
+	 * SP_EL0 set yet. We set it here for the first time pointing to the
+	 * beginning of the user accessible part of the stack (the top).
+	 */
+	msr	sp_el0, x1
+
+	/*
+	 * Set SP_EL1 to point at the end of the ESF. Since we have relocated
+	 * the ESF at the beginning of the privileged stack area, when the ESF
+	 * is popped out by z_arm64_exit_exc() the SP_EL1 will be at the right
+	 * location for when the next exception will come.
+	 */
 	mov	sp, x0
 	b	z_arm64_exit_exc

arch/arm64/core/vector_table.S

@@ -25,10 +25,6 @@ _ASM_FILE_PROLOGUE
  */
 
 .macro z_arm64_enter_exc xreg0, xreg1
-
-	/* Switch to SP_EL0 */
-	msr	spsel, #0
-
 	/*
 	 * Two things can happen to the remaining registers:
 	 *

arch/arm64/include/kernel_arch_func.h

@@ -40,7 +40,7 @@ static inline void arch_switch(void *switch_to, void **switched_from)
 }
 
 extern void z_arm64_fatal_error(z_arch_esf_t *esf, unsigned int reason);
-extern void z_arm64_userspace_enter(z_arch_esf_t *esf);
+extern void z_arm64_userspace_enter(z_arch_esf_t *esf, uintptr_t sp_el0);
 extern void z_arm64_set_ttbr0(uintptr_t ttbr0);
 extern void z_arm64_ptable_ipi(void);
 

arch/arm64/include/offsets_short_arch.h

@@ -9,9 +9,4 @@
 
 #include <offsets.h>
 
-#ifdef CONFIG_USERSPACE
-#define _thread_offset_to_priv_stack_start \
-	(___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_start_OFFSET)
-#endif
-
 #endif /* ZEPHYR_ARCH_ARM64_INCLUDE_OFFSETS_SHORT_ARCH_H_ */

include/arch/arm64/thread.h

@@ -33,16 +33,17 @@ struct _callee_saved {
 	uint64_t x26;
 	uint64_t x27;
 	uint64_t x28;
-	uint64_t x29; /* FP */
-	uint64_t sp;
+	uint64_t x29;
+	uint64_t sp_el0;
+	uint64_t sp_elx;
+	uint64_t xzr;
 };
 
 typedef struct _callee_saved _callee_saved_t;
 
 struct _thread_arch {
 #ifdef CONFIG_USERSPACE
 	struct arm_mmu_ptables *ptables;
-	uint64_t priv_stack_start;
 #endif
 };
 

include/arch/arm64/thread_stack.h

@@ -18,6 +18,8 @@
 #endif
 
 /*
+ * [ see also comments in arch/arm64/core/thread.c ]
+ *
  * High memory addresses
  *
  * +-------------------+ <- thread.stack_info.start + thread.stack_info.size
@@ -31,7 +33,7 @@
  * |   Unused stack    |
  * |                   |
  * +-------------------+ <- thread.stack_info.start
- * |  Reserved memory  | } K_(THREAD|KERNEL)_STACK_RESERVED
+ * | Privileged stack  | } K_(THREAD|KERNEL)_STACK_RESERVED
  * +-------------------+ <- thread.stack_obj
  *
  * Low Memory addresses

subsys/debug/thread_info.c

@@ -51,8 +51,9 @@ size_t _kernel_thread_info_offsets[] = {
 						   user_options),
 	[THREAD_INFO_OFFSET_T_PRIO] = offsetof(struct _thread_base, prio),
 #if defined(CONFIG_ARM64)
+	/* We are assuming that the SP of interest is SP_EL1 */
 	[THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread,
-						callee_saved.sp),
+						callee_saved.sp_elx),
 #elif defined(CONFIG_ARM)
 	[THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread,
 						callee_saved.psp),