![Frederic Weisbecker](/assets/img/avatar_default.png)
Impact: Provide a way to pause the function graph tracer As suggested by Steven Rostedt, the previous patch that prevented from spinlock function tracing shouldn't use the raw_spinlock to fix it. It's much better to follow lockdep with normal spinlock, so this patch adds a new flag for each task to make the function graph tracer able to be paused. We also can send an ftrace_printk whithout worrying of the irrelevant traced spinlock during insertion. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
538 lines
13 KiB
C
538 lines
13 KiB
C
/*
|
|
* Code for replacing ftrace calls with jumps.
|
|
*
|
|
* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
|
|
*
|
|
* Thanks goes to Ingo Molnar, for suggesting the idea.
|
|
* Mathieu Desnoyers, for suggesting postponing the modifications.
|
|
* Arjan van de Ven, for keeping me straight, and explaining to me
|
|
* the dangers of modifying code on the run.
|
|
*/
|
|
|
|
#include <linux/spinlock.h>
|
|
#include <linux/hardirq.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/ftrace.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/init.h>
|
|
#include <linux/list.h>
|
|
|
|
#include <asm/ftrace.h>
|
|
#include <linux/ftrace.h>
|
|
#include <asm/nops.h>
|
|
#include <asm/nmi.h>
|
|
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
union ftrace_code_union {
|
|
char code[MCOUNT_INSN_SIZE];
|
|
struct {
|
|
char e8;
|
|
int offset;
|
|
} __attribute__((packed));
|
|
};
|
|
|
|
static int ftrace_calc_offset(long ip, long addr)
|
|
{
|
|
return (int)(addr - ip);
|
|
}
|
|
|
|
static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
|
|
{
|
|
static union ftrace_code_union calc;
|
|
|
|
calc.e8 = 0xe8;
|
|
calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
|
|
|
|
/*
|
|
* No locking needed, this must be called via kstop_machine
|
|
* which in essence is like running on a uniprocessor machine.
|
|
*/
|
|
return calc.code;
|
|
}
|
|
|
|
/*
|
|
* Modifying code must take extra care. On an SMP machine, if
|
|
* the code being modified is also being executed on another CPU
|
|
* that CPU will have undefined results and possibly take a GPF.
|
|
* We use kstop_machine to stop other CPUS from exectuing code.
|
|
* But this does not stop NMIs from happening. We still need
|
|
* to protect against that. We separate out the modification of
|
|
* the code to take care of this.
|
|
*
|
|
* Two buffers are added: An IP buffer and a "code" buffer.
|
|
*
|
|
* 1) Put the instruction pointer into the IP buffer
|
|
* and the new code into the "code" buffer.
|
|
* 2) Set a flag that says we are modifying code
|
|
* 3) Wait for any running NMIs to finish.
|
|
* 4) Write the code
|
|
* 5) clear the flag.
|
|
* 6) Wait for any running NMIs to finish.
|
|
*
|
|
* If an NMI is executed, the first thing it does is to call
|
|
* "ftrace_nmi_enter". This will check if the flag is set to write
|
|
* and if it is, it will write what is in the IP and "code" buffers.
|
|
*
|
|
* The trick is, it does not matter if everyone is writing the same
|
|
* content to the code location. Also, if a CPU is executing code
|
|
* it is OK to write to that code location if the contents being written
|
|
* are the same as what exists.
|
|
*/
|
|
|
|
static atomic_t in_nmi = ATOMIC_INIT(0);
|
|
static int mod_code_status; /* holds return value of text write */
|
|
static int mod_code_write; /* set when NMI should do the write */
|
|
static void *mod_code_ip; /* holds the IP to write to */
|
|
static void *mod_code_newcode; /* holds the text to write to the IP */
|
|
|
|
static unsigned nmi_wait_count;
|
|
static atomic_t nmi_update_count = ATOMIC_INIT(0);
|
|
|
|
int ftrace_arch_read_dyn_info(char *buf, int size)
|
|
{
|
|
int r;
|
|
|
|
r = snprintf(buf, size, "%u %u",
|
|
nmi_wait_count,
|
|
atomic_read(&nmi_update_count));
|
|
return r;
|
|
}
|
|
|
|
static void ftrace_mod_code(void)
|
|
{
|
|
/*
|
|
* Yes, more than one CPU process can be writing to mod_code_status.
|
|
* (and the code itself)
|
|
* But if one were to fail, then they all should, and if one were
|
|
* to succeed, then they all should.
|
|
*/
|
|
mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
|
|
MCOUNT_INSN_SIZE);
|
|
}
|
|
|
|
void ftrace_nmi_enter(void)
|
|
{
|
|
atomic_inc(&in_nmi);
|
|
/* Must have in_nmi seen before reading write flag */
|
|
smp_mb();
|
|
if (mod_code_write) {
|
|
ftrace_mod_code();
|
|
atomic_inc(&nmi_update_count);
|
|
}
|
|
}
|
|
|
|
void ftrace_nmi_exit(void)
|
|
{
|
|
/* Finish all executions before clearing in_nmi */
|
|
smp_wmb();
|
|
atomic_dec(&in_nmi);
|
|
}
|
|
|
|
static void wait_for_nmi(void)
|
|
{
|
|
int waited = 0;
|
|
|
|
while (atomic_read(&in_nmi)) {
|
|
waited = 1;
|
|
cpu_relax();
|
|
}
|
|
|
|
if (waited)
|
|
nmi_wait_count++;
|
|
}
|
|
|
|
static int
|
|
do_ftrace_mod_code(unsigned long ip, void *new_code)
|
|
{
|
|
mod_code_ip = (void *)ip;
|
|
mod_code_newcode = new_code;
|
|
|
|
/* The buffers need to be visible before we let NMIs write them */
|
|
smp_wmb();
|
|
|
|
mod_code_write = 1;
|
|
|
|
/* Make sure write bit is visible before we wait on NMIs */
|
|
smp_mb();
|
|
|
|
wait_for_nmi();
|
|
|
|
/* Make sure all running NMIs have finished before we write the code */
|
|
smp_mb();
|
|
|
|
ftrace_mod_code();
|
|
|
|
/* Make sure the write happens before clearing the bit */
|
|
smp_wmb();
|
|
|
|
mod_code_write = 0;
|
|
|
|
/* make sure NMIs see the cleared bit */
|
|
smp_mb();
|
|
|
|
wait_for_nmi();
|
|
|
|
return mod_code_status;
|
|
}
|
|
|
|
|
|
|
|
|
|
static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
|
|
|
|
static unsigned char *ftrace_nop_replace(void)
|
|
{
|
|
return ftrace_nop;
|
|
}
|
|
|
|
static int
|
|
ftrace_modify_code(unsigned long ip, unsigned char *old_code,
|
|
unsigned char *new_code)
|
|
{
|
|
unsigned char replaced[MCOUNT_INSN_SIZE];
|
|
|
|
/*
|
|
* Note: Due to modules and __init, code can
|
|
* disappear and change, we need to protect against faulting
|
|
* as well as code changing. We do this by using the
|
|
* probe_kernel_* functions.
|
|
*
|
|
* No real locking needed, this code is run through
|
|
* kstop_machine, or before SMP starts.
|
|
*/
|
|
|
|
/* read the text we want to modify */
|
|
if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
|
|
return -EFAULT;
|
|
|
|
/* Make sure it is what we expect it to be */
|
|
if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
|
|
return -EINVAL;
|
|
|
|
/* replace the text with the new text */
|
|
if (do_ftrace_mod_code(ip, new_code))
|
|
return -EPERM;
|
|
|
|
sync_core();
|
|
|
|
return 0;
|
|
}
|
|
|
|
int ftrace_make_nop(struct module *mod,
|
|
struct dyn_ftrace *rec, unsigned long addr)
|
|
{
|
|
unsigned char *new, *old;
|
|
unsigned long ip = rec->ip;
|
|
|
|
old = ftrace_call_replace(ip, addr);
|
|
new = ftrace_nop_replace();
|
|
|
|
return ftrace_modify_code(rec->ip, old, new);
|
|
}
|
|
|
|
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
|
{
|
|
unsigned char *new, *old;
|
|
unsigned long ip = rec->ip;
|
|
|
|
old = ftrace_nop_replace();
|
|
new = ftrace_call_replace(ip, addr);
|
|
|
|
return ftrace_modify_code(rec->ip, old, new);
|
|
}
|
|
|
|
int ftrace_update_ftrace_func(ftrace_func_t func)
|
|
{
|
|
unsigned long ip = (unsigned long)(&ftrace_call);
|
|
unsigned char old[MCOUNT_INSN_SIZE], *new;
|
|
int ret;
|
|
|
|
memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
|
|
new = ftrace_call_replace(ip, (unsigned long)func);
|
|
ret = ftrace_modify_code(ip, old, new);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int __init ftrace_dyn_arch_init(void *data)
|
|
{
|
|
extern const unsigned char ftrace_test_p6nop[];
|
|
extern const unsigned char ftrace_test_nop5[];
|
|
extern const unsigned char ftrace_test_jmp[];
|
|
int faulted = 0;
|
|
|
|
/*
|
|
* There is no good nop for all x86 archs.
|
|
* We will default to using the P6_NOP5, but first we
|
|
* will test to make sure that the nop will actually
|
|
* work on this CPU. If it faults, we will then
|
|
* go to a lesser efficient 5 byte nop. If that fails
|
|
* we then just use a jmp as our nop. This isn't the most
|
|
* efficient nop, but we can not use a multi part nop
|
|
* since we would then risk being preempted in the middle
|
|
* of that nop, and if we enabled tracing then, it might
|
|
* cause a system crash.
|
|
*
|
|
* TODO: check the cpuid to determine the best nop.
|
|
*/
|
|
asm volatile (
|
|
"ftrace_test_jmp:"
|
|
"jmp ftrace_test_p6nop\n"
|
|
"nop\n"
|
|
"nop\n"
|
|
"nop\n" /* 2 byte jmp + 3 bytes */
|
|
"ftrace_test_p6nop:"
|
|
P6_NOP5
|
|
"jmp 1f\n"
|
|
"ftrace_test_nop5:"
|
|
".byte 0x66,0x66,0x66,0x66,0x90\n"
|
|
"1:"
|
|
".section .fixup, \"ax\"\n"
|
|
"2: movl $1, %0\n"
|
|
" jmp ftrace_test_nop5\n"
|
|
"3: movl $2, %0\n"
|
|
" jmp 1b\n"
|
|
".previous\n"
|
|
_ASM_EXTABLE(ftrace_test_p6nop, 2b)
|
|
_ASM_EXTABLE(ftrace_test_nop5, 3b)
|
|
: "=r"(faulted) : "0" (faulted));
|
|
|
|
switch (faulted) {
|
|
case 0:
|
|
pr_info("ftrace: converting mcount calls to 0f 1f 44 00 00\n");
|
|
memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
|
|
break;
|
|
case 1:
|
|
pr_info("ftrace: converting mcount calls to 66 66 66 66 90\n");
|
|
memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
|
|
break;
|
|
case 2:
|
|
pr_info("ftrace: converting mcount calls to jmp . + 5\n");
|
|
memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
|
|
break;
|
|
}
|
|
|
|
/* The return code is retured via data */
|
|
*(unsigned long *)data = 0;
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
extern void ftrace_graph_call(void);
|
|
|
|
static int ftrace_mod_jmp(unsigned long ip,
|
|
int old_offset, int new_offset)
|
|
{
|
|
unsigned char code[MCOUNT_INSN_SIZE];
|
|
|
|
if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
|
|
return -EFAULT;
|
|
|
|
if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
|
|
return -EINVAL;
|
|
|
|
*(int *)(&code[1]) = new_offset;
|
|
|
|
if (do_ftrace_mod_code(ip, &code))
|
|
return -EPERM;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int ftrace_enable_ftrace_graph_caller(void)
|
|
{
|
|
unsigned long ip = (unsigned long)(&ftrace_graph_call);
|
|
int old_offset, new_offset;
|
|
|
|
old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
|
|
new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
|
|
|
|
return ftrace_mod_jmp(ip, old_offset, new_offset);
|
|
}
|
|
|
|
int ftrace_disable_ftrace_graph_caller(void)
|
|
{
|
|
unsigned long ip = (unsigned long)(&ftrace_graph_call);
|
|
int old_offset, new_offset;
|
|
|
|
old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
|
|
new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
|
|
|
|
return ftrace_mod_jmp(ip, old_offset, new_offset);
|
|
}
|
|
|
|
#else /* CONFIG_DYNAMIC_FTRACE */
|
|
|
|
/*
|
|
* These functions are picked from those used on
|
|
* this page for dynamic ftrace. They have been
|
|
* simplified to ignore all traces in NMI context.
|
|
*/
|
|
static atomic_t in_nmi;
|
|
|
|
void ftrace_nmi_enter(void)
|
|
{
|
|
atomic_inc(&in_nmi);
|
|
}
|
|
|
|
void ftrace_nmi_exit(void)
|
|
{
|
|
atomic_dec(&in_nmi);
|
|
}
|
|
|
|
#endif /* !CONFIG_DYNAMIC_FTRACE */
|
|
|
|
/* Add a function return address to the trace stack on thread info.*/
|
|
static int push_return_trace(unsigned long ret, unsigned long long time,
|
|
unsigned long func, int *depth)
|
|
{
|
|
int index;
|
|
|
|
if (!current->ret_stack)
|
|
return -EBUSY;
|
|
|
|
/* The return trace stack is full */
|
|
if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
|
|
atomic_inc(¤t->trace_overrun);
|
|
return -EBUSY;
|
|
}
|
|
|
|
index = ++current->curr_ret_stack;
|
|
barrier();
|
|
current->ret_stack[index].ret = ret;
|
|
current->ret_stack[index].func = func;
|
|
current->ret_stack[index].calltime = time;
|
|
*depth = index;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Retrieve a function return address to the trace stack on thread info.*/
|
|
static void pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
|
|
{
|
|
int index;
|
|
|
|
index = current->curr_ret_stack;
|
|
|
|
if (unlikely(index < 0)) {
|
|
ftrace_graph_stop();
|
|
WARN_ON(1);
|
|
/* Might as well panic, otherwise we have no where to go */
|
|
*ret = (unsigned long)panic;
|
|
return;
|
|
}
|
|
|
|
*ret = current->ret_stack[index].ret;
|
|
trace->func = current->ret_stack[index].func;
|
|
trace->calltime = current->ret_stack[index].calltime;
|
|
trace->overrun = atomic_read(¤t->trace_overrun);
|
|
trace->depth = index;
|
|
barrier();
|
|
current->curr_ret_stack--;
|
|
|
|
}
|
|
|
|
/*
|
|
* Send the trace to the ring-buffer.
|
|
* @return the original return address.
|
|
*/
|
|
unsigned long ftrace_return_to_handler(void)
|
|
{
|
|
struct ftrace_graph_ret trace;
|
|
unsigned long ret;
|
|
|
|
pop_return_trace(&trace, &ret);
|
|
trace.rettime = cpu_clock(raw_smp_processor_id());
|
|
ftrace_graph_return(&trace);
|
|
|
|
if (unlikely(!ret)) {
|
|
ftrace_graph_stop();
|
|
WARN_ON(1);
|
|
/* Might as well panic. What else to do? */
|
|
ret = (unsigned long)panic;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Hook the return address and push it in the stack of return addrs
|
|
* in current thread info.
|
|
*/
|
|
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
|
|
{
|
|
unsigned long old;
|
|
unsigned long long calltime;
|
|
int faulted;
|
|
struct ftrace_graph_ent trace;
|
|
unsigned long return_hooker = (unsigned long)
|
|
&return_to_handler;
|
|
|
|
/* Nmi's are currently unsupported */
|
|
if (unlikely(atomic_read(&in_nmi)))
|
|
return;
|
|
|
|
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
|
|
return;
|
|
|
|
/*
|
|
* Protect against fault, even if it shouldn't
|
|
* happen. This tool is too much intrusive to
|
|
* ignore such a protection.
|
|
*/
|
|
asm volatile(
|
|
"1: " _ASM_MOV " (%[parent_old]), %[old]\n"
|
|
"2: " _ASM_MOV " %[return_hooker], (%[parent_replaced])\n"
|
|
" movl $0, %[faulted]\n"
|
|
|
|
".section .fixup, \"ax\"\n"
|
|
"3: movl $1, %[faulted]\n"
|
|
".previous\n"
|
|
|
|
_ASM_EXTABLE(1b, 3b)
|
|
_ASM_EXTABLE(2b, 3b)
|
|
|
|
: [parent_replaced] "=r" (parent), [old] "=r" (old),
|
|
[faulted] "=r" (faulted)
|
|
: [parent_old] "0" (parent), [return_hooker] "r" (return_hooker)
|
|
: "memory"
|
|
);
|
|
|
|
if (unlikely(faulted)) {
|
|
ftrace_graph_stop();
|
|
WARN_ON(1);
|
|
return;
|
|
}
|
|
|
|
if (unlikely(!__kernel_text_address(old))) {
|
|
ftrace_graph_stop();
|
|
*parent = old;
|
|
WARN_ON(1);
|
|
return;
|
|
}
|
|
|
|
calltime = cpu_clock(raw_smp_processor_id());
|
|
|
|
if (push_return_trace(old, calltime,
|
|
self_addr, &trace.depth) == -EBUSY) {
|
|
*parent = old;
|
|
return;
|
|
}
|
|
|
|
trace.func = self_addr;
|
|
|
|
/* Only trace if the calling function expects to */
|
|
if (!ftrace_graph_entry(&trace)) {
|
|
current->curr_ret_stack--;
|
|
*parent = old;
|
|
}
|
|
}
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|