kernel: move current thread pointer management to core code

Define the generic _current directly and get rid of the generic arch_current_get(). The SMP default implementation is now known as z_smp_current_get(). It is no longer inlined which saves significant binary size (about 10% for some random test case I checked). Introduce z_current_thread_set() and use it in place of arch_current_thread_set() for updating the current thread pointer given this is not necessarily an architecture specific operation. The architecture specific optimization, when enabled, should only care about its own things and not have to also update the generic _current_cpu->current copy. Signed-off-by: Nicolas Pitre <[email protected]>
zephyrproject-rtos · Jan 10, 2025 · 7a3124d · 7a3124d
1 parent 46aa671
commit 7a3124d
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 60 deletions.
diff --git a/arch/arm/core/cortex_m/thread.c b/arch/arm/core/cortex_m/thread.c
@@ -522,7 +522,7 @@ void arch_switch_to_main_thread(struct k_thread *main_thread, char *stack_ptr,
 {
 	z_arm_prepare_switch_to_main();
 
-	arch_current_thread_set(main_thread);
+	z_current_thread_set(main_thread);
 
 #if defined(CONFIG_THREAD_LOCAL_STORAGE)
 	/* On Cortex-M, TLS uses a global variable as pointer to

diff --git a/arch/posix/core/swap.c b/arch/posix/core/swap.c
@@ -50,7 +50,7 @@ int arch_swap(unsigned int key)
 		_current->callee_saved.thread_status;
 
 
-	arch_current_thread_set(_kernel.ready_q.cache);
+	z_current_thread_set(_kernel.ready_q.cache);
 #if CONFIG_INSTRUMENT_THREAD_SWITCHING
 	z_thread_mark_switched_in();
 #endif
@@ -94,7 +94,7 @@ void arch_switch_to_main_thread(struct k_thread *main_thread, char *stack_ptr,
 	z_thread_mark_switched_out();
 #endif
 
-	arch_current_thread_set(_kernel.ready_q.cache);
+	z_current_thread_set(_kernel.ready_q.cache);
 
 #ifdef CONFIG_INSTRUMENT_THREAD_SWITCHING
 	z_thread_mark_switched_in();

diff --git a/include/zephyr/arch/arch_inlines.h b/include/zephyr/arch/arch_inlines.h
@@ -34,6 +34,4 @@
 #include <zephyr/arch/sparc/arch_inlines.h>
 #endif
 
-#include <zephyr/arch/common/arch_inlines.h>
-
 #endif /* ZEPHYR_INCLUDE_ARCH_INLINES_H_ */
diff --git a/include/zephyr/arch/common/arch_inlines.h b/include/zephyr/arch/common/arch_inlines.h
diff --git a/include/zephyr/arch/riscv/arch_inlines.h b/include/zephyr/arch/riscv/arch_inlines.h
@@ -28,13 +28,12 @@ static ALWAYS_INLINE _cpu_t *arch_curr_cpu(void)
 }
 
 #ifdef CONFIG_RISCV_CURRENT_VIA_GP
+
 register struct k_thread *__arch_current_thread __asm__("gp");
 
 #define arch_current_thread() __arch_current_thread
-#define arch_current_thread_set(thread)                                                            \
-	do {                                                                                       \
-		__arch_current_thread = _current_cpu->current = (thread);                          \
-	} while (0)
+#define arch_current_thread_set(thread) ({ __arch_current_thread = (thread); })
+
 #endif /* CONFIG_RISCV_CURRENT_VIA_GP */
 
 static ALWAYS_INLINE unsigned int arch_num_cpus(void)

diff --git a/include/zephyr/kernel_structs.h b/include/zephyr/kernel_structs.h
@@ -260,16 +260,28 @@ extern atomic_t _cpus_active;
  * another SMP CPU.
  */
 bool z_smp_cpu_mobile(void);
-
 #define _current_cpu ({ __ASSERT_NO_MSG(!z_smp_cpu_mobile()); \
 			arch_curr_cpu(); })
-#define _current arch_current_thread()
+
+struct k_thread *z_smp_current_get(void);
+#define _current z_smp_current_get()
 
 #else
 #define _current_cpu (&_kernel.cpus[0])
 #define _current _kernel.cpus[0].current
 #endif
 
+/* This is always invoked from a context where preemption is disabled */
+#define z_current_thread_set(thread) ({ _current_cpu->current = (thread); })
+
+#ifdef CONFIG_ARCH_HAS_CUSTOM_CURRENT_IMPL
+#undef _current
+#define _current arch_current_thread()
+#undef z_current_thread_set
+#define z_current_thread_set(thread) \
+	arch_current_thread_set(({ _current_cpu->current = (thread); }))
+#endif
+
 /* kernel wait queue record */
 #ifdef CONFIG_WAITQ_SCALABLE
 

diff --git a/kernel/include/kswap.h b/kernel/include/kswap.h
@@ -133,7 +133,7 @@ static ALWAYS_INLINE unsigned int do_swap(unsigned int key,
 #endif /* CONFIG_SMP */
 		z_thread_mark_switched_out();
 		z_sched_switch_spin(new_thread);
-		arch_current_thread_set(new_thread);
+		z_current_thread_set(new_thread);
 
 #ifdef CONFIG_TIMESLICING
 		z_reset_time_slice(new_thread);
@@ -259,6 +259,6 @@ static inline void z_dummy_thread_init(struct k_thread *dummy_thread)
 	dummy_thread->base.slice_ticks = 0;
 #endif /* CONFIG_TIMESLICE_PER_THREAD */
 
-	arch_current_thread_set(dummy_thread);
+	z_current_thread_set(dummy_thread);
 }
 #endif /* ZEPHYR_KERNEL_INCLUDE_KSWAP_H_ */
diff --git a/kernel/sched.c b/kernel/sched.c
@@ -797,11 +797,11 @@ struct k_thread *z_swap_next_thread(void)
 }
 
 #ifdef CONFIG_USE_SWITCH
-/* Just a wrapper around arch_current_thread_set(xxx) with tracing */
+/* Just a wrapper around z_current_thread_set(xxx) with tracing */
 static inline void set_current(struct k_thread *new_thread)
 {
 	z_thread_mark_switched_out();
-	arch_current_thread_set(new_thread);
+	z_current_thread_set(new_thread);
 }
 
 /**

diff --git a/kernel/smp.c b/kernel/smp.c
@@ -248,3 +248,17 @@ bool z_smp_cpu_mobile(void)
 	arch_irq_unlock(k);
 	return !pinned;
 }
+
+struct k_thread *z_smp_current_get(void)
+{
+	/*
+	 * _current is a field read from _current_cpu, which can race
+	 * with preemption before it is read.  We must lock local
+	 * interrupts when reading it.
+	 */
+	unsigned int key = arch_irq_lock();
+	struct k_thread *t = _current_cpu->current;
+
+	arch_irq_unlock(key);
+	return t;
+}