2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* SMP support for ppc.
|
|
|
|
*
|
|
|
|
* Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
|
|
|
|
* deal of code from the sparc and intel versions.
|
|
|
|
*
|
|
|
|
* Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
|
|
|
|
*
|
|
|
|
* PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
|
|
|
|
* Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#undef DEBUG
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/sysdev.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/notifier.h>
|
2005-12-12 19:56:47 +00:00
|
|
|
#include <linux/topology.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#include <asm/ptrace.h>
|
|
|
|
#include <asm/atomic.h>
|
|
|
|
#include <asm/irq.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/prom.h>
|
|
|
|
#include <asm/smp.h>
|
|
|
|
#include <asm/time.h>
|
|
|
|
#include <asm/machdep.h>
|
2008-07-27 05:24:52 +00:00
|
|
|
#include <asm/cputhreads.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/cputable.h>
|
|
|
|
#include <asm/system.h>
|
2005-09-27 03:51:59 +00:00
|
|
|
#include <asm/mpic.h>
|
2005-11-11 10:15:21 +00:00
|
|
|
#include <asm/vdso_datapage.h>
|
2005-11-04 23:33:55 +00:00
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
#include <asm/paca.h>
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef DEBUG
|
2005-11-15 04:16:38 +00:00
|
|
|
#include <asm/udbg.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#define DBG(fmt...) udbg_printf(fmt)
|
|
|
|
#else
|
|
|
|
#define DBG(fmt...)
|
|
|
|
#endif
|
|
|
|
|
2011-03-08 03:40:04 +00:00
|
|
|
|
|
|
|
/* Store all idle threads, this can be reused instead of creating
|
|
|
|
* a new thread. Also avoids complicated thread destroy functionality
|
|
|
|
* for idle threads.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
/*
|
|
|
|
* Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
|
|
|
|
* removed after init for !CONFIG_HOTPLUG_CPU.
|
|
|
|
*/
|
|
|
|
static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
|
|
|
|
#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
|
|
|
|
#define set_idle_for_cpu(x, p) (per_cpu(idle_thread_array, x) = (p))
|
|
|
|
#else
|
|
|
|
static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
|
|
|
|
#define get_idle_for_cpu(x) (idle_thread_array[(x)])
|
|
|
|
#define set_idle_for_cpu(x, p) (idle_thread_array[(x)] = (p))
|
|
|
|
#endif
|
|
|
|
|
2005-11-15 04:16:38 +00:00
|
|
|
struct thread_info *secondary_ti;
|
|
|
|
|
2010-04-26 15:32:41 +00:00
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
|
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-16 08:24:05 +00:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
|
2008-07-27 05:24:53 +00:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_core_map);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-11-04 23:33:55 +00:00
|
|
|
/* SMP operations for this machine */
|
2005-04-16 22:20:36 +00:00
|
|
|
struct smp_ops_t *smp_ops;
|
|
|
|
|
2009-06-18 23:30:07 +00:00
|
|
|
/* Can't be static due to PowerMac hackery */
|
|
|
|
volatile unsigned int cpu_callin_map[NR_CPUS];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
int smt_enabled_at_boot = 1;
|
|
|
|
|
2005-12-04 07:39:43 +00:00
|
|
|
static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;
|
|
|
|
|
2005-11-04 23:33:55 +00:00
|
|
|
#ifdef CONFIG_PPC64
|
2011-04-11 21:46:19 +00:00
|
|
|
int __devinit smp_generic_kick_cpu(int nr)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
BUG_ON(nr < 0 || nr >= NR_CPUS);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The processor is currently spinning, waiting for the
|
|
|
|
* cpu_start field to become non-zero After we set cpu_start,
|
|
|
|
* the processor will continue on to secondary_start
|
|
|
|
*/
|
|
|
|
paca[nr].cpu_start = 1;
|
2005-05-01 15:58:47 +00:00
|
|
|
smp_mb();
|
2011-04-11 21:46:19 +00:00
|
|
|
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-11-04 23:33:55 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
IRQ: Maintain regs pointer globally rather than passing to IRQ handlers
Maintain a per-CPU global "struct pt_regs *" variable which can be used instead
of passing regs around manually through all ~1800 interrupt handlers in the
Linux kernel.
The regs pointer is used in few places, but it potentially costs both stack
space and code to pass it around. On the FRV arch, removing the regs parameter
from all the genirq function results in a 20% speed up of the IRQ exit path
(ie: from leaving timer_interrupt() to leaving do_IRQ()).
Where appropriate, an arch may override the generic storage facility and do
something different with the variable. On FRV, for instance, the address is
maintained in GR28 at all times inside the kernel as part of general exception
handling.
Having looked over the code, it appears that the parameter may be handed down
through up to twenty or so layers of functions. Consider a USB character
device attached to a USB hub, attached to a USB controller that posts its
interrupts through a cascaded auxiliary interrupt controller. A character
device driver may want to pass regs to the sysrq handler through the input
layer which adds another few layers of parameter passing.
I've build this code with allyesconfig for x86_64 and i386. I've runtested the
main part of the code on FRV and i386, though I can't test most of the drivers.
I've also done partial conversion for powerpc and MIPS - these at least compile
with minimal configurations.
This will affect all archs. Mostly the changes should be relatively easy.
Take do_IRQ(), store the regs pointer at the beginning, saving the old one:
struct pt_regs *old_regs = set_irq_regs(regs);
And put the old one back at the end:
set_irq_regs(old_regs);
Don't pass regs through to generic_handle_irq() or __do_IRQ().
In timer_interrupt(), this sort of change will be necessary:
- update_process_times(user_mode(regs));
- profile_tick(CPU_PROFILING, regs);
+ update_process_times(user_mode(get_irq_regs()));
+ profile_tick(CPU_PROFILING);
I'd like to move update_process_times()'s use of get_irq_regs() into itself,
except that i386, alone of the archs, uses something other than user_mode().
Some notes on the interrupt handling in the drivers:
(*) input_dev() is now gone entirely. The regs pointer is no longer stored in
the input_dev struct.
(*) finish_unlinks() in drivers/usb/host/ohci-q.c needs checking. It does
something different depending on whether it's been supplied with a regs
pointer or not.
(*) Various IRQ handler function pointers have been moved to type
irq_handler_t.
Signed-Off-By: David Howells <dhowells@redhat.com>
(cherry picked from 1b16e7ac850969f38b375e511e3fa2f474a33867 commit)
2006-10-05 13:55:46 +00:00
|
|
|
void smp_message_recv(int msg)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
switch(msg) {
|
|
|
|
case PPC_MSG_CALL_FUNCTION:
|
2008-06-26 09:22:13 +00:00
|
|
|
generic_smp_call_function_interrupt();
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
2005-11-04 23:33:55 +00:00
|
|
|
case PPC_MSG_RESCHEDULE:
|
2008-10-10 01:56:45 +00:00
|
|
|
/* we notice need_resched on exit */
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
2008-06-26 09:22:13 +00:00
|
|
|
case PPC_MSG_CALL_FUNC_SINGLE:
|
|
|
|
generic_smp_call_function_single_interrupt();
|
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
case PPC_MSG_DEBUGGER_BREAK:
|
2005-12-04 07:39:43 +00:00
|
|
|
if (crash_ipi_function_ptr) {
|
IRQ: Maintain regs pointer globally rather than passing to IRQ handlers
Maintain a per-CPU global "struct pt_regs *" variable which can be used instead
of passing regs around manually through all ~1800 interrupt handlers in the
Linux kernel.
The regs pointer is used in few places, but it potentially costs both stack
space and code to pass it around. On the FRV arch, removing the regs parameter
from all the genirq function results in a 20% speed up of the IRQ exit path
(ie: from leaving timer_interrupt() to leaving do_IRQ()).
Where appropriate, an arch may override the generic storage facility and do
something different with the variable. On FRV, for instance, the address is
maintained in GR28 at all times inside the kernel as part of general exception
handling.
Having looked over the code, it appears that the parameter may be handed down
through up to twenty or so layers of functions. Consider a USB character
device attached to a USB hub, attached to a USB controller that posts its
interrupts through a cascaded auxiliary interrupt controller. A character
device driver may want to pass regs to the sysrq handler through the input
layer which adds another few layers of parameter passing.
I've build this code with allyesconfig for x86_64 and i386. I've runtested the
main part of the code on FRV and i386, though I can't test most of the drivers.
I've also done partial conversion for powerpc and MIPS - these at least compile
with minimal configurations.
This will affect all archs. Mostly the changes should be relatively easy.
Take do_IRQ(), store the regs pointer at the beginning, saving the old one:
struct pt_regs *old_regs = set_irq_regs(regs);
And put the old one back at the end:
set_irq_regs(old_regs);
Don't pass regs through to generic_handle_irq() or __do_IRQ().
In timer_interrupt(), this sort of change will be necessary:
- update_process_times(user_mode(regs));
- profile_tick(CPU_PROFILING, regs);
+ update_process_times(user_mode(get_irq_regs()));
+ profile_tick(CPU_PROFILING);
I'd like to move update_process_times()'s use of get_irq_regs() into itself,
except that i386, alone of the archs, uses something other than user_mode().
Some notes on the interrupt handling in the drivers:
(*) input_dev() is now gone entirely. The regs pointer is no longer stored in
the input_dev struct.
(*) finish_unlinks() in drivers/usb/host/ohci-q.c needs checking. It does
something different depending on whether it's been supplied with a regs
pointer or not.
(*) Various IRQ handler function pointers have been moved to type
irq_handler_t.
Signed-Off-By: David Howells <dhowells@redhat.com>
(cherry picked from 1b16e7ac850969f38b375e511e3fa2f474a33867 commit)
2006-10-05 13:55:46 +00:00
|
|
|
crash_ipi_function_ptr(get_irq_regs());
|
2005-12-04 07:39:43 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
#ifdef CONFIG_DEBUGGER
|
IRQ: Maintain regs pointer globally rather than passing to IRQ handlers
Maintain a per-CPU global "struct pt_regs *" variable which can be used instead
of passing regs around manually through all ~1800 interrupt handlers in the
Linux kernel.
The regs pointer is used in few places, but it potentially costs both stack
space and code to pass it around. On the FRV arch, removing the regs parameter
from all the genirq function results in a 20% speed up of the IRQ exit path
(ie: from leaving timer_interrupt() to leaving do_IRQ()).
Where appropriate, an arch may override the generic storage facility and do
something different with the variable. On FRV, for instance, the address is
maintained in GR28 at all times inside the kernel as part of general exception
handling.
Having looked over the code, it appears that the parameter may be handed down
through up to twenty or so layers of functions. Consider a USB character
device attached to a USB hub, attached to a USB controller that posts its
interrupts through a cascaded auxiliary interrupt controller. A character
device driver may want to pass regs to the sysrq handler through the input
layer which adds another few layers of parameter passing.
I've build this code with allyesconfig for x86_64 and i386. I've runtested the
main part of the code on FRV and i386, though I can't test most of the drivers.
I've also done partial conversion for powerpc and MIPS - these at least compile
with minimal configurations.
This will affect all archs. Mostly the changes should be relatively easy.
Take do_IRQ(), store the regs pointer at the beginning, saving the old one:
struct pt_regs *old_regs = set_irq_regs(regs);
And put the old one back at the end:
set_irq_regs(old_regs);
Don't pass regs through to generic_handle_irq() or __do_IRQ().
In timer_interrupt(), this sort of change will be necessary:
- update_process_times(user_mode(regs));
- profile_tick(CPU_PROFILING, regs);
+ update_process_times(user_mode(get_irq_regs()));
+ profile_tick(CPU_PROFILING);
I'd like to move update_process_times()'s use of get_irq_regs() into itself,
except that i386, alone of the archs, uses something other than user_mode().
Some notes on the interrupt handling in the drivers:
(*) input_dev() is now gone entirely. The regs pointer is no longer stored in
the input_dev struct.
(*) finish_unlinks() in drivers/usb/host/ohci-q.c needs checking. It does
something different depending on whether it's been supplied with a regs
pointer or not.
(*) Various IRQ handler function pointers have been moved to type
irq_handler_t.
Signed-Off-By: David Howells <dhowells@redhat.com>
(cherry picked from 1b16e7ac850969f38b375e511e3fa2f474a33867 commit)
2006-10-05 13:55:46 +00:00
|
|
|
debugger_ipi(get_irq_regs());
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
2005-12-04 07:39:43 +00:00
|
|
|
#endif /* CONFIG_DEBUGGER */
|
|
|
|
/* FALLTHROUGH */
|
2005-04-16 22:20:36 +00:00
|
|
|
default:
|
|
|
|
printk("SMP %d: smp_message_recv(): unknown msg %d\n",
|
|
|
|
smp_processor_id(), msg);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-11-14 20:11:49 +00:00
|
|
|
static irqreturn_t call_function_action(int irq, void *data)
|
|
|
|
{
|
|
|
|
generic_smp_call_function_interrupt();
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t reschedule_action(int irq, void *data)
|
|
|
|
{
|
|
|
|
/* we just need the return path side effect of checking need_resched */
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t call_function_single_action(int irq, void *data)
|
|
|
|
{
|
|
|
|
generic_smp_call_function_single_interrupt();
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t debug_ipi_action(int irq, void *data)
|
|
|
|
{
|
|
|
|
smp_message_recv(PPC_MSG_DEBUGGER_BREAK);
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irq_handler_t smp_ipi_action[] = {
|
|
|
|
[PPC_MSG_CALL_FUNCTION] = call_function_action,
|
|
|
|
[PPC_MSG_RESCHEDULE] = reschedule_action,
|
|
|
|
[PPC_MSG_CALL_FUNC_SINGLE] = call_function_single_action,
|
|
|
|
[PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action,
|
|
|
|
};
|
|
|
|
|
|
|
|
const char *smp_ipi_name[] = {
|
|
|
|
[PPC_MSG_CALL_FUNCTION] = "ipi call function",
|
|
|
|
[PPC_MSG_RESCHEDULE] = "ipi reschedule",
|
|
|
|
[PPC_MSG_CALL_FUNC_SINGLE] = "ipi call function single",
|
|
|
|
[PPC_MSG_DEBUGGER_BREAK] = "ipi debugger",
|
|
|
|
};
|
|
|
|
|
|
|
|
/* optional function to request ipi, for controllers with >= 4 ipis */
|
|
|
|
int smp_request_message_ipi(int virq, int msg)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC)
|
|
|
|
if (msg == PPC_MSG_DEBUGGER_BREAK) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
err = request_irq(virq, smp_ipi_action[msg], IRQF_DISABLED|IRQF_PERCPU,
|
|
|
|
smp_ipi_name[msg], 0);
|
|
|
|
WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
|
|
|
|
virq, smp_ipi_name[msg], err);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void smp_send_reschedule(int cpu)
|
|
|
|
{
|
2006-07-04 04:09:36 +00:00
|
|
|
if (likely(smp_ops))
|
|
|
|
smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-06-26 09:22:13 +00:00
|
|
|
void arch_send_call_function_single_ipi(int cpu)
|
|
|
|
{
|
|
|
|
smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE);
|
|
|
|
}
|
|
|
|
|
2009-09-24 15:34:45 +00:00
|
|
|
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
|
2008-06-26 09:22:13 +00:00
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
2009-09-24 15:34:45 +00:00
|
|
|
for_each_cpu(cpu, mask)
|
2008-06-26 09:22:13 +00:00
|
|
|
smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_DEBUGGER
|
|
|
|
void smp_send_debugger_break(int cpu)
|
|
|
|
{
|
2006-07-04 04:09:36 +00:00
|
|
|
if (likely(smp_ops))
|
|
|
|
smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2005-12-04 07:39:43 +00:00
|
|
|
#ifdef CONFIG_KEXEC
|
|
|
|
void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
|
|
|
|
{
|
|
|
|
crash_ipi_function_ptr = crash_ipi_callback;
|
2006-07-04 04:09:36 +00:00
|
|
|
if (crash_ipi_callback && smp_ops) {
|
2005-12-04 07:39:43 +00:00
|
|
|
mb();
|
|
|
|
smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static void stop_this_cpu(void *dummy)
|
|
|
|
{
|
2009-11-25 11:48:52 +00:00
|
|
|
/* Remove this CPU */
|
|
|
|
set_cpu_online(smp_processor_id(), false);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
local_irq_disable();
|
|
|
|
while (1)
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
2007-09-17 23:43:40 +00:00
|
|
|
void smp_send_stop(void)
|
|
|
|
{
|
2008-06-06 09:18:06 +00:00
|
|
|
smp_call_function(stop_this_cpu, NULL, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct thread_info *current_set[NR_CPUS];
|
|
|
|
|
|
|
|
static void __devinit smp_store_cpu_info(int id)
|
|
|
|
{
|
2009-10-29 13:34:14 +00:00
|
|
|
per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __init smp_prepare_cpus(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
DBG("smp_prepare_cpus\n");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* setup_cpu may need to be called on the boot cpu. We havent
|
|
|
|
* spun any cpus up but lets be paranoid.
|
|
|
|
*/
|
|
|
|
BUG_ON(boot_cpuid != smp_processor_id());
|
|
|
|
|
|
|
|
/* Fixup boot cpu */
|
|
|
|
smp_store_cpu_info(boot_cpuid);
|
|
|
|
cpu_callin_map[boot_cpuid] = 1;
|
|
|
|
|
2010-04-26 15:32:41 +00:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
|
|
|
|
GFP_KERNEL, cpu_to_node(cpu));
|
|
|
|
zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
|
|
|
|
GFP_KERNEL, cpu_to_node(cpu));
|
|
|
|
}
|
|
|
|
|
|
|
|
cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
|
|
|
|
cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
|
|
|
|
|
2006-07-04 04:09:36 +00:00
|
|
|
if (smp_ops)
|
2009-09-08 17:38:52 +00:00
|
|
|
if (smp_ops->probe)
|
|
|
|
max_cpus = smp_ops->probe();
|
|
|
|
else
|
|
|
|
max_cpus = NR_CPUS;
|
2006-07-04 04:09:36 +00:00
|
|
|
else
|
|
|
|
max_cpus = 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __devinit smp_prepare_boot_cpu(void)
|
|
|
|
{
|
|
|
|
BUG_ON(smp_processor_id() != boot_cpuid);
|
2005-11-04 23:33:55 +00:00
|
|
|
#ifdef CONFIG_PPC64
|
2005-04-16 22:20:36 +00:00
|
|
|
paca[boot_cpuid].__current = current;
|
2005-11-04 23:33:55 +00:00
|
|
|
#endif
|
2006-01-12 09:06:01 +00:00
|
|
|
current_set[boot_cpuid] = task_thread_info(current);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
/* State of each CPU during hotplug phases */
|
2011-03-31 22:23:37 +00:00
|
|
|
static DEFINE_PER_CPU(int, cpu_state) = { 0 };
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
int generic_cpu_disable(void)
|
|
|
|
{
|
|
|
|
unsigned int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (cpu == boot_cpuid)
|
|
|
|
return -EBUSY;
|
|
|
|
|
2009-09-24 15:34:48 +00:00
|
|
|
set_cpu_online(cpu, false);
|
2005-11-10 02:37:51 +00:00
|
|
|
#ifdef CONFIG_PPC64
|
2005-11-11 10:15:21 +00:00
|
|
|
vdso_data->processorCount--;
|
2005-11-10 03:26:12 +00:00
|
|
|
#endif
|
2011-02-11 02:05:17 +00:00
|
|
|
migrate_irqs();
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void generic_cpu_die(unsigned int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < 100; i++) {
|
2005-05-01 15:58:47 +00:00
|
|
|
smp_rmb();
|
2005-04-16 22:20:36 +00:00
|
|
|
if (per_cpu(cpu_state, cpu) == CPU_DEAD)
|
|
|
|
return;
|
|
|
|
msleep(100);
|
|
|
|
}
|
|
|
|
printk(KERN_ERR "CPU%d didn't die...\n", cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
void generic_mach_cpu_die(void)
|
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
local_irq_disable();
|
2011-02-10 07:46:50 +00:00
|
|
|
idle_task_exit();
|
2005-04-16 22:20:36 +00:00
|
|
|
cpu = smp_processor_id();
|
|
|
|
printk(KERN_DEBUG "CPU%d offline\n", cpu);
|
|
|
|
__get_cpu_var(cpu_state) = CPU_DEAD;
|
2005-05-01 15:58:47 +00:00
|
|
|
smp_wmb();
|
2005-04-16 22:20:36 +00:00
|
|
|
while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
|
|
|
|
cpu_relax();
|
|
|
|
}
|
2011-03-31 22:23:37 +00:00
|
|
|
|
|
|
|
void generic_set_cpu_dead(unsigned int cpu)
|
|
|
|
{
|
|
|
|
per_cpu(cpu_state, cpu) = CPU_DEAD;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2011-03-08 03:40:04 +00:00
|
|
|
struct create_idle {
|
|
|
|
struct work_struct work;
|
|
|
|
struct task_struct *idle;
|
|
|
|
struct completion done;
|
|
|
|
int cpu;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void __cpuinit do_fork_idle(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct create_idle *c_idle =
|
|
|
|
container_of(work, struct create_idle, work);
|
|
|
|
|
|
|
|
c_idle->idle = fork_idle(c_idle->cpu);
|
|
|
|
complete(&c_idle->done);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __cpuinit create_idle(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct thread_info *ti;
|
|
|
|
struct create_idle c_idle = {
|
|
|
|
.cpu = cpu,
|
|
|
|
.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
|
|
|
|
};
|
|
|
|
INIT_WORK_ONSTACK(&c_idle.work, do_fork_idle);
|
|
|
|
|
|
|
|
c_idle.idle = get_idle_for_cpu(cpu);
|
|
|
|
|
|
|
|
/* We can't use kernel_thread since we must avoid to
|
|
|
|
* reschedule the child. We use a workqueue because
|
|
|
|
* we want to fork from a kernel thread, not whatever
|
|
|
|
* userspace process happens to be trying to online us.
|
|
|
|
*/
|
|
|
|
if (!c_idle.idle) {
|
|
|
|
schedule_work(&c_idle.work);
|
|
|
|
wait_for_completion(&c_idle.done);
|
|
|
|
} else
|
|
|
|
init_idle(c_idle.idle, cpu);
|
|
|
|
if (IS_ERR(c_idle.idle)) {
|
|
|
|
pr_err("Failed fork for CPU %u: %li", cpu, PTR_ERR(c_idle.idle));
|
|
|
|
return PTR_ERR(c_idle.idle);
|
|
|
|
}
|
|
|
|
ti = task_thread_info(c_idle.idle);
|
|
|
|
|
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
paca[cpu].__current = c_idle.idle;
|
|
|
|
paca[cpu].kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
|
|
|
|
#endif
|
|
|
|
ti->cpu = cpu;
|
|
|
|
current_set[cpu] = ti;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-01-11 07:15:34 +00:00
|
|
|
int __cpuinit __cpu_up(unsigned int cpu)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-03-08 03:40:04 +00:00
|
|
|
int rc, c;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-11-04 23:33:55 +00:00
|
|
|
secondary_ti = current_set[cpu];
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-07-04 04:09:36 +00:00
|
|
|
if (smp_ops == NULL ||
|
|
|
|
(smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2011-03-08 03:40:04 +00:00
|
|
|
/* Make sure we have an idle thread */
|
|
|
|
rc = create_idle(cpu);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Make sure callin-map entry is 0 (can be leftover a CPU
|
|
|
|
* hotplug
|
|
|
|
*/
|
|
|
|
cpu_callin_map[cpu] = 0;
|
|
|
|
|
|
|
|
/* The information for processor bringup must
|
|
|
|
* be written out to main store before we release
|
|
|
|
* the processor.
|
|
|
|
*/
|
2005-05-01 15:58:47 +00:00
|
|
|
smp_mb();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* wake up cpus */
|
|
|
|
DBG("smp: kicking cpu %d\n", cpu);
|
2011-04-11 21:46:19 +00:00
|
|
|
rc = smp_ops->kick_cpu(cpu);
|
|
|
|
if (rc) {
|
|
|
|
pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
|
|
|
|
return rc;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* wait to see if the cpu made a callin (is actually up).
|
|
|
|
* use this value that I found through experimentation.
|
|
|
|
* -- Cort
|
|
|
|
*/
|
|
|
|
if (system_state < SYSTEM_RUNNING)
|
2006-06-17 22:52:44 +00:00
|
|
|
for (c = 50000; c && !cpu_callin_map[cpu]; c--)
|
2005-04-16 22:20:36 +00:00
|
|
|
udelay(100);
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
else
|
|
|
|
/*
|
|
|
|
* CPUs can take much longer to come up in the
|
|
|
|
* hotplug case. Wait five seconds.
|
|
|
|
*/
|
2009-06-23 23:26:37 +00:00
|
|
|
for (c = 5000; c && !cpu_callin_map[cpu]; c--)
|
|
|
|
msleep(1);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!cpu_callin_map[cpu]) {
|
2010-08-04 18:28:34 +00:00
|
|
|
printk(KERN_ERR "Processor %u is stuck.\n", cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOENT;
|
|
|
|
}
|
|
|
|
|
2010-08-04 18:28:34 +00:00
|
|
|
DBG("Processor %u found.\n", cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (smp_ops->give_timebase)
|
|
|
|
smp_ops->give_timebase();
|
|
|
|
|
|
|
|
/* Wait until cpu puts itself in the online map */
|
|
|
|
while (!cpu_online(cpu))
|
|
|
|
cpu_relax();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-07-27 05:24:54 +00:00
|
|
|
/* Return the value of the reg property corresponding to the given
|
|
|
|
* logical cpu.
|
|
|
|
*/
|
|
|
|
int cpu_to_core_id(int cpu)
|
|
|
|
{
|
|
|
|
struct device_node *np;
|
|
|
|
const int *reg;
|
|
|
|
int id = -1;
|
|
|
|
|
|
|
|
np = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (!np)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
reg = of_get_property(np, "reg", NULL);
|
|
|
|
if (!reg)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
id = *reg;
|
|
|
|
out:
|
|
|
|
of_node_put(np);
|
|
|
|
return id;
|
|
|
|
}
|
|
|
|
|
2010-10-06 08:36:59 +00:00
|
|
|
/* Helper routines for cpu to core mapping */
|
|
|
|
int cpu_core_index_of_thread(int cpu)
|
|
|
|
{
|
|
|
|
return cpu >> threads_shift;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
|
|
|
|
|
|
|
|
int cpu_first_thread_of_core(int core)
|
|
|
|
{
|
|
|
|
return core << threads_shift;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
|
|
|
|
|
|
|
|
/* Must be called when no change can occur to cpu_present_map,
|
2008-07-27 05:24:53 +00:00
|
|
|
* i.e. during cpu online or offline.
|
|
|
|
*/
|
|
|
|
static struct device_node *cpu_to_l2cache(int cpu)
|
|
|
|
{
|
|
|
|
struct device_node *np;
|
2008-12-10 20:16:07 +00:00
|
|
|
struct device_node *cache;
|
2008-07-27 05:24:53 +00:00
|
|
|
|
|
|
|
if (!cpu_present(cpu))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
np = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (np == NULL)
|
|
|
|
return NULL;
|
|
|
|
|
2008-12-10 20:16:07 +00:00
|
|
|
cache = of_find_next_cache_node(np);
|
|
|
|
|
2008-07-27 05:24:53 +00:00
|
|
|
of_node_put(np);
|
|
|
|
|
2008-12-10 20:16:07 +00:00
|
|
|
return cache;
|
2008-07-27 05:24:53 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Activate a secondary processor. */
|
2011-02-10 07:45:24 +00:00
|
|
|
void __devinit start_secondary(void *unused)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
unsigned int cpu = smp_processor_id();
|
2008-07-27 05:24:53 +00:00
|
|
|
struct device_node *l2_cache;
|
2008-07-27 05:24:52 +00:00
|
|
|
int i, base;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
atomic_inc(&init_mm.mm_count);
|
|
|
|
current->active_mm = &init_mm;
|
|
|
|
|
|
|
|
smp_store_cpu_info(cpu);
|
2005-11-04 23:33:55 +00:00
|
|
|
set_dec(tb_ticks_per_jiffy);
|
2005-11-09 23:45:30 +00:00
|
|
|
preempt_disable();
|
2005-04-16 22:20:36 +00:00
|
|
|
cpu_callin_map[cpu] = 1;
|
|
|
|
|
2009-09-08 17:38:52 +00:00
|
|
|
if (smp_ops->setup_cpu)
|
|
|
|
smp_ops->setup_cpu(cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (smp_ops->take_timebase)
|
|
|
|
smp_ops->take_timebase();
|
|
|
|
|
2007-09-21 03:26:03 +00:00
|
|
|
secondary_cpu_time_init();
|
|
|
|
|
2011-03-08 03:49:33 +00:00
|
|
|
#ifdef CONFIG_PPC64
|
|
|
|
if (system_state == SYSTEM_RUNNING)
|
|
|
|
vdso_data->processorCount++;
|
|
|
|
#endif
|
2008-06-26 09:22:13 +00:00
|
|
|
ipi_call_lock();
|
2008-09-07 14:57:22 +00:00
|
|
|
notify_cpu_starting(cpu);
|
2009-09-24 15:34:48 +00:00
|
|
|
set_cpu_online(cpu, true);
|
2008-07-27 05:24:52 +00:00
|
|
|
/* Update sibling maps */
|
2010-10-06 08:36:59 +00:00
|
|
|
base = cpu_first_thread_sibling(cpu);
|
2008-07-27 05:24:52 +00:00
|
|
|
for (i = 0; i < threads_per_core; i++) {
|
|
|
|
if (cpu_is_offline(base + i))
|
|
|
|
continue;
|
2010-04-26 15:32:41 +00:00
|
|
|
cpumask_set_cpu(cpu, cpu_sibling_mask(base + i));
|
|
|
|
cpumask_set_cpu(base + i, cpu_sibling_mask(cpu));
|
2008-07-27 05:24:53 +00:00
|
|
|
|
|
|
|
/* cpu_core_map should be a superset of
|
|
|
|
* cpu_sibling_map even if we don't have cache
|
|
|
|
* information, so update the former here, too.
|
|
|
|
*/
|
2010-04-26 15:32:41 +00:00
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(base + i));
|
|
|
|
cpumask_set_cpu(base + i, cpu_core_mask(cpu));
|
2008-07-27 05:24:52 +00:00
|
|
|
}
|
2008-07-27 05:24:53 +00:00
|
|
|
l2_cache = cpu_to_l2cache(cpu);
|
|
|
|
for_each_online_cpu(i) {
|
|
|
|
struct device_node *np = cpu_to_l2cache(i);
|
|
|
|
if (!np)
|
|
|
|
continue;
|
|
|
|
if (np == l2_cache) {
|
2010-04-26 15:32:41 +00:00
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_set_cpu(i, cpu_core_mask(cpu));
|
2008-07-27 05:24:53 +00:00
|
|
|
}
|
|
|
|
of_node_put(np);
|
|
|
|
}
|
|
|
|
of_node_put(l2_cache);
|
2008-06-26 09:22:13 +00:00
|
|
|
ipi_call_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
cpu_idle();
|
2011-02-10 07:45:24 +00:00
|
|
|
|
|
|
|
BUG();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int setup_profiling_timer(unsigned int multiplier)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __init smp_cpus_done(unsigned int max_cpus)
|
|
|
|
{
|
2010-04-26 15:32:34 +00:00
|
|
|
cpumask_var_t old_mask;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* We want the setup_cpu() here to be called from CPU 0, but our
|
|
|
|
* init thread may have been "borrowed" by another CPU in the meantime
|
|
|
|
* se we pin us down to CPU 0 for a short while
|
|
|
|
*/
|
2010-04-26 15:32:34 +00:00
|
|
|
alloc_cpumask_var(&old_mask, GFP_NOWAIT);
|
|
|
|
cpumask_copy(old_mask, ¤t->cpus_allowed);
|
2010-03-26 12:03:29 +00:00
|
|
|
set_cpus_allowed_ptr(current, cpumask_of(boot_cpuid));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-09-08 17:38:52 +00:00
|
|
|
if (smp_ops && smp_ops->setup_cpu)
|
2006-07-04 04:09:36 +00:00
|
|
|
smp_ops->setup_cpu(boot_cpuid);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-04-26 15:32:34 +00:00
|
|
|
set_cpus_allowed_ptr(current, old_mask);
|
|
|
|
|
|
|
|
free_cpumask_var(old_mask);
|
2005-12-12 19:56:47 +00:00
|
|
|
|
2011-03-08 02:50:37 +00:00
|
|
|
if (smp_ops && smp_ops->bringup_done)
|
|
|
|
smp_ops->bringup_done();
|
|
|
|
|
2005-12-12 19:56:47 +00:00
|
|
|
dump_numa_cpu_topology();
|
2011-03-08 02:50:37 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2010-08-10 20:02:05 +00:00
|
|
|
int arch_sd_sibling_asym_packing(void)
|
|
|
|
{
|
|
|
|
if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
|
|
|
|
printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
|
|
|
|
return SD_ASYM_PACKING;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
int __cpu_disable(void)
|
|
|
|
{
|
2008-07-27 05:24:53 +00:00
|
|
|
struct device_node *l2_cache;
|
2008-07-27 05:24:52 +00:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int base, i;
|
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-27 05:24:52 +00:00
|
|
|
if (!smp_ops->cpu_disable)
|
|
|
|
return -ENOSYS;
|
|
|
|
|
|
|
|
err = smp_ops->cpu_disable();
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
/* Update sibling maps */
|
2010-10-06 08:36:59 +00:00
|
|
|
base = cpu_first_thread_sibling(cpu);
|
2008-07-27 05:24:52 +00:00
|
|
|
for (i = 0; i < threads_per_core; i++) {
|
2010-04-26 15:32:41 +00:00
|
|
|
cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i));
|
|
|
|
cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu));
|
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(base + i));
|
|
|
|
cpumask_clear_cpu(base + i, cpu_core_mask(cpu));
|
2008-07-27 05:24:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
l2_cache = cpu_to_l2cache(cpu);
|
|
|
|
for_each_present_cpu(i) {
|
|
|
|
struct device_node *np = cpu_to_l2cache(i);
|
|
|
|
if (!np)
|
|
|
|
continue;
|
|
|
|
if (np == l2_cache) {
|
2010-04-26 15:32:41 +00:00
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(i));
|
|
|
|
cpumask_clear_cpu(i, cpu_core_mask(cpu));
|
2008-07-27 05:24:53 +00:00
|
|
|
}
|
|
|
|
of_node_put(np);
|
2008-07-27 05:24:52 +00:00
|
|
|
}
|
2008-07-27 05:24:53 +00:00
|
|
|
of_node_put(l2_cache);
|
|
|
|
|
2008-07-27 05:24:52 +00:00
|
|
|
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __cpu_die(unsigned int cpu)
|
|
|
|
{
|
|
|
|
if (smp_ops->cpu_die)
|
|
|
|
smp_ops->cpu_die(cpu);
|
|
|
|
}
|
2010-01-14 09:52:44 +00:00
|
|
|
|
|
|
|
static DEFINE_MUTEX(powerpc_cpu_hotplug_driver_mutex);
|
|
|
|
|
|
|
|
void cpu_hotplug_driver_lock()
|
|
|
|
{
|
|
|
|
mutex_lock(&powerpc_cpu_hotplug_driver_mutex);
|
|
|
|
}
|
|
|
|
|
|
|
|
void cpu_hotplug_driver_unlock()
|
|
|
|
{
|
|
|
|
mutex_unlock(&powerpc_cpu_hotplug_driver_mutex);
|
|
|
|
}
|
2010-05-19 02:56:29 +00:00
|
|
|
|
|
|
|
void cpu_die(void)
|
|
|
|
{
|
|
|
|
if (ppc_md.cpu_die)
|
|
|
|
ppc_md.cpu_die();
|
2011-02-10 07:45:24 +00:00
|
|
|
|
|
|
|
/* If we return, we re-enter start_secondary */
|
|
|
|
start_secondary_resume();
|
2010-05-19 02:56:29 +00:00
|
|
|
}
|
2011-02-10 07:45:24 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|