|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH RFC 11/13] samples: ftrace: Maintain Tasks RCU trampoline nesting in direct-call trampolines
Follow the register_ftrace_direct() contract in the sample modules: on
x86-64 and arm64, have each hand-written trampoline increment
current->rcu_tramp_nesting before calling its C handler and decrement it
before returning, via a small shared samples/ftrace/ftrace-direct.h.
%r11 and x12/w13 are used as scratch; both are caller-saved, non-argument
registers and therefore dead on entry to and exit from an fentry
trampoline.
The header pulls in the generated asm-offsets.h only on those two
architectures, since it is not generally safe to include from C (PPC32's
TASK_SIZE and arm64's TRAMP_VALIAS clash with the C definitions; the
latter is worked around locally with push_macro/pop_macro). Other
architectures get empty macros and are unchanged.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@xxxxxxxxxxxxxx>
---
samples/ftrace/ftrace-direct-modify.c | 9 ++++
samples/ftrace/ftrace-direct-multi-modify.c | 9 ++++
samples/ftrace/ftrace-direct-multi.c | 5 +++
samples/ftrace/ftrace-direct-too.c | 5 +++
samples/ftrace/ftrace-direct.c | 5 +++
samples/ftrace/ftrace-direct.h | 64 +++++++++++++++++++++++++++++
6 files changed, 97 insertions(+)
diff --git a/samples/ftrace/ftrace-direct-modify.c
b/samples/ftrace/ftrace-direct-modify.c
index 164d9dd6fd92..eb8230fa4242 100644
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -73,7 +74,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" call my_direct_func1\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -85,7 +88,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" call my_direct_func2\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -141,11 +146,13 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func1\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -153,11 +160,13 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func2\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi-modify.c
b/samples/ftrace/ftrace-direct-multi-modify.c
index b03766c6217b..c8f1062e5d1a 100644
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -77,10 +78,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func1\n"
" popq %rdi\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -92,10 +95,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func2\n"
" popq %rdi\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -154,6 +159,7 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -162,6 +168,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -169,6 +176,7 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -177,6 +185,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi.c
b/samples/ftrace/ftrace-direct-multi.c
index 3fe6ddaf0b69..bc6a88dd4ffc 100644
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#include <linux/sched/stat.h>
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
@@ -56,10 +57,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -101,6 +104,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -109,6 +113,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-too.c
b/samples/ftrace/ftrace-direct-too.c
index bf2411aa6fd7..247e418644a2 100644
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -61,6 +62,7 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" pushq %rdi\n"
" pushq %rsi\n"
" pushq %rdx\n"
@@ -70,6 +72,7 @@ asm (
" popq %rdx\n"
" popq %rsi\n"
" popq %rdi\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -110,6 +113,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #48\n"
" stp x9, x30, [sp]\n"
" stp x0, x1, [sp, #16]\n"
@@ -119,6 +123,7 @@ asm (
" ldp x0, x1, [sp, #16]\n"
" ldp x2, x3, [sp, #32]\n"
" add sp, sp, #48\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
index 5368c8c39cbb..9e1964baf28b 100644
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c
@@ -3,6 +3,7 @@
#include <linux/sched.h> /* for wake_up_process() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -54,9 +55,11 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ RCU_TASKS_TRAMP_ENTER
" pushq %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ RCU_TASKS_TRAMP_EXIT
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -97,6 +100,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ RCU_TASKS_TRAMP_ENTER
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -104,6 +108,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ RCU_TASKS_TRAMP_EXIT
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.h b/samples/ftrace/ftrace-direct.h
new file mode 100644
index 000000000000..d0313f33f47f
--- /dev/null
+++ b/samples/ftrace/ftrace-direct.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _SAMPLES_FTRACE_DIRECT_H
+#define _SAMPLES_FTRACE_DIRECT_H
+
+#include <linux/stringify.h>
+
+/*
+ * A direct-call trampoline is entered with no lock, refcount or RCU marker
+ * held; only Tasks RCU keeps it (and, for a module, its text) alive while a
+ * task is inside it or preempted in something it called. On architectures
+ * that select ARCH_HAS_RCU_TASKS_PREEMPT_QS a preemption is a Tasks RCU
+ * quiescent state unless current->rcu_tramp_nesting is non-zero, so the
+ * trampoline must raise it before calling out and drop it afterwards, exactly
+ * like the ftrace and BPF trampolines do. See rcu_tasks_trampoline_enter()
+ * and register_ftrace_direct(). The instructions before the increment and
+ * after the decrement are covered by ftrace_direct_mark_module().
+ *
+ * These expand to instruction strings for use inside the samples' asm()
+ * trampolines. The scratch register is caller-saved and not an argument
+ * register, so it is dead on entry to and exit from an fentry trampoline.
+ *
+ * The generated asm-offsets.h is only pulled in on the architectures that need
+ * it here: it is not generally safe to include from C (e.g. PPC32's TASK_SIZE
+ * and arm64's TRAMP_VALIAS clash with the C definitions), which is why the
+ * samples themselves guard their own include of it.
+ */
+#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_X86_64)
+
+#include <asm/asm-offsets.h>
+
+#define RCU_TASKS_TRAMP_ENTER \
+ " movq %gs:current_task(%rip), %r11\n"
\
+ " incl " __stringify(TASK_rcu_tramp_nesting) "(%r11)\n"
+#define RCU_TASKS_TRAMP_EXIT \
+ " movq %gs:current_task(%rip), %r11\n"
\
+ " decl " __stringify(TASK_rcu_tramp_nesting) "(%r11)\n"
+
+#elif defined(CONFIG_TASKS_RCU) && defined(CONFIG_ARM64)
+
+/* arm64's asm-offsets.h redefines TRAMP_VALIAS from <asm/fixmap.h>. */
+#pragma push_macro("TRAMP_VALIAS")
+#undef TRAMP_VALIAS
+#include <asm/asm-offsets.h>
+#pragma pop_macro("TRAMP_VALIAS")
+
+#define RCU_TASKS_TRAMP_ENTER \
+ " mrs x12, sp_el0\n"
\
+ " ldr w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n"
\
+ " add w13, w13, #1\n"
\
+ " str w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n"
+#define RCU_TASKS_TRAMP_EXIT \
+ " mrs x12, sp_el0\n"
\
+ " ldr w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n"
\
+ " sub w13, w13, #1\n"
\
+ " str w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n"
+
+#else
+
+#define RCU_TASKS_TRAMP_ENTER
+#define RCU_TASKS_TRAMP_EXIT
+
+#endif
+
+#endif /* _SAMPLES_FTRACE_DIRECT_H */
--
2.55.0
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |