From 1bd858a507937cef6e8ca5efc6820a5744e73683 Mon Sep 17 00:00:00 2001 From: Anil S Keshavamurthy Date: Mon, 26 Jun 2006 00:25:25 -0700 Subject: [PATCH] [PATCH] Notify page fault call chain for x86_64 Currently in the do_page_fault() code path, we call notify_die(DIE_PAGE_FAULT, ...) to notify the page fault. Since notify_die() is highly overloaded, this page fault notification is currently being sent to all the components registered with register_die_notification() which uses the same die_chain to loop for all the registered components which is unnecessary. In order to optimize the do_page_fault() code path, this critical page fault notification is now moved to different call chain and the test results showed great improvements. And the kprobes which is interested in this notifications, now registers onto this new call chain only when it need to, i.e Kprobes now registers for page fault notification only when their are an active probes and unregisters from this page fault notification when no probes are active. I have incorporated all the feedback given by Ananth and Keith and everyone, and thanks for all the review feedback. This patch: Overloading of page fault notification with the notify_die() has performance issues(since the only interested components for page fault is kprobes and/or kdb) and hence this patch introduces the new notifier call chain exclusively for page fault notifications their by avoiding notifying unnecessary components in the do_page_fault() code path. Signed-off-by: Anil S Keshavamurthy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/x86_64/mm/fault.c | 39 +++++++++++++++++++++++++++++++++++-- include/asm-x86_64/kdebug.h | 2 ++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/arch/x86_64/mm/fault.c b/arch/x86_64/mm/fault.c index 55250593d8c..0803d3858af 100644 --- a/arch/x86_64/mm/fault.c +++ b/arch/x86_64/mm/fault.c @@ -41,6 +41,41 @@ #define PF_RSVD (1<<3) #define PF_INSTR (1<<4) +#ifdef CONFIG_KPROBES +ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain); + +/* Hook to register for page fault notifications */ +int register_page_fault_notifier(struct notifier_block *nb) +{ + vmalloc_sync_all(); + return atomic_notifier_chain_register(¬ify_page_fault_chain, nb); +} + +int unregister_page_fault_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_unregister(¬ify_page_fault_chain, nb); +} + +static inline int notify_page_fault(enum die_val val, const char *str, + struct pt_regs *regs, long err, int trap, int sig) +{ + struct die_args args = { + .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig + }; + return atomic_notifier_call_chain(¬ify_page_fault_chain, val, &args); +} +#else +static inline int notify_page_fault(enum die_val val, const char *str, + struct pt_regs *regs, long err, int trap, int sig) +{ + return NOTIFY_DONE; +} +#endif + void bust_spinlocks(int yes) { int loglevel_save = console_loglevel; @@ -348,7 +383,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, if (vmalloc_fault(address) >= 0) return; } - if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14, + if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14, SIGSEGV) == NOTIFY_STOP) return; /* @@ -358,7 +393,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, goto bad_area_nosemaphore; } - if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14, + if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14, SIGSEGV) == NOTIFY_STOP) return; diff --git a/include/asm-x86_64/kdebug.h b/include/asm-x86_64/kdebug.h index cf795631d9b..cd52c7f33bc 100644 --- a/include/asm-x86_64/kdebug.h +++ b/include/asm-x86_64/kdebug.h @@ -15,6 +15,8 @@ struct die_args { extern int register_die_notifier(struct notifier_block *); extern int unregister_die_notifier(struct notifier_block *); +extern int register_page_fault_notifier(struct notifier_block *); +extern int unregister_page_fault_notifier(struct notifier_block *); extern struct atomic_notifier_head die_chain; /* Grossly misnamed. */