Merge branch 'x86/paravirt' into x86/apic
Conflicts: arch/x86/mach-voyager/voyager_smp.c
This commit is contained in:
commit
eca217b36e
27 changed files with 1401 additions and 1169 deletions
|
@ -12,21 +12,38 @@
|
||||||
#define CLBR_EAX (1 << 0)
|
#define CLBR_EAX (1 << 0)
|
||||||
#define CLBR_ECX (1 << 1)
|
#define CLBR_ECX (1 << 1)
|
||||||
#define CLBR_EDX (1 << 2)
|
#define CLBR_EDX (1 << 2)
|
||||||
|
#define CLBR_EDI (1 << 3)
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_32
|
||||||
#define CLBR_RSI (1 << 3)
|
/* CLBR_ANY should match all regs platform has. For i386, that's just it */
|
||||||
#define CLBR_RDI (1 << 4)
|
#define CLBR_ANY ((1 << 4) - 1)
|
||||||
|
|
||||||
|
#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
|
||||||
|
#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
|
||||||
|
#define CLBR_SCRATCH (0)
|
||||||
|
#else
|
||||||
|
#define CLBR_RAX CLBR_EAX
|
||||||
|
#define CLBR_RCX CLBR_ECX
|
||||||
|
#define CLBR_RDX CLBR_EDX
|
||||||
|
#define CLBR_RDI CLBR_EDI
|
||||||
|
#define CLBR_RSI (1 << 4)
|
||||||
#define CLBR_R8 (1 << 5)
|
#define CLBR_R8 (1 << 5)
|
||||||
#define CLBR_R9 (1 << 6)
|
#define CLBR_R9 (1 << 6)
|
||||||
#define CLBR_R10 (1 << 7)
|
#define CLBR_R10 (1 << 7)
|
||||||
#define CLBR_R11 (1 << 8)
|
#define CLBR_R11 (1 << 8)
|
||||||
|
|
||||||
#define CLBR_ANY ((1 << 9) - 1)
|
#define CLBR_ANY ((1 << 9) - 1)
|
||||||
|
|
||||||
|
#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
|
||||||
|
CLBR_RCX | CLBR_R8 | CLBR_R9)
|
||||||
|
#define CLBR_RET_REG (CLBR_RAX)
|
||||||
|
#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
|
||||||
|
|
||||||
#include <asm/desc_defs.h>
|
#include <asm/desc_defs.h>
|
||||||
#else
|
|
||||||
/* CLBR_ANY should match all regs platform has. For i386, that's just it */
|
|
||||||
#define CLBR_ANY ((1 << 3) - 1)
|
|
||||||
#endif /* X86_64 */
|
#endif /* X86_64 */
|
||||||
|
|
||||||
|
#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
|
||||||
|
|
||||||
#ifndef __ASSEMBLY__
|
#ifndef __ASSEMBLY__
|
||||||
#include <linux/types.h>
|
#include <linux/types.h>
|
||||||
#include <linux/cpumask.h>
|
#include <linux/cpumask.h>
|
||||||
|
@ -40,6 +57,14 @@ struct tss_struct;
|
||||||
struct mm_struct;
|
struct mm_struct;
|
||||||
struct desc_struct;
|
struct desc_struct;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Wrapper type for pointers to code which uses the non-standard
|
||||||
|
* calling convention. See PV_CALL_SAVE_REGS_THUNK below.
|
||||||
|
*/
|
||||||
|
struct paravirt_callee_save {
|
||||||
|
void *func;
|
||||||
|
};
|
||||||
|
|
||||||
/* general info */
|
/* general info */
|
||||||
struct pv_info {
|
struct pv_info {
|
||||||
unsigned int kernel_rpl;
|
unsigned int kernel_rpl;
|
||||||
|
@ -189,11 +214,15 @@ struct pv_irq_ops {
|
||||||
* expected to use X86_EFLAGS_IF; all other bits
|
* expected to use X86_EFLAGS_IF; all other bits
|
||||||
* returned from save_fl are undefined, and may be ignored by
|
* returned from save_fl are undefined, and may be ignored by
|
||||||
* restore_fl.
|
* restore_fl.
|
||||||
|
*
|
||||||
|
* NOTE: These functions callers expect the callee to preserve
|
||||||
|
* more registers than the standard C calling convention.
|
||||||
*/
|
*/
|
||||||
unsigned long (*save_fl)(void);
|
struct paravirt_callee_save save_fl;
|
||||||
void (*restore_fl)(unsigned long);
|
struct paravirt_callee_save restore_fl;
|
||||||
void (*irq_disable)(void);
|
struct paravirt_callee_save irq_disable;
|
||||||
void (*irq_enable)(void);
|
struct paravirt_callee_save irq_enable;
|
||||||
|
|
||||||
void (*safe_halt)(void);
|
void (*safe_halt)(void);
|
||||||
void (*halt)(void);
|
void (*halt)(void);
|
||||||
|
|
||||||
|
@ -279,11 +308,11 @@ struct pv_mmu_ops {
|
||||||
void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
|
void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
|
||||||
pte_t *ptep, pte_t pte);
|
pte_t *ptep, pte_t pte);
|
||||||
|
|
||||||
pteval_t (*pte_val)(pte_t);
|
struct paravirt_callee_save pte_val;
|
||||||
pte_t (*make_pte)(pteval_t pte);
|
struct paravirt_callee_save make_pte;
|
||||||
|
|
||||||
pgdval_t (*pgd_val)(pgd_t);
|
struct paravirt_callee_save pgd_val;
|
||||||
pgd_t (*make_pgd)(pgdval_t pgd);
|
struct paravirt_callee_save make_pgd;
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS >= 3
|
#if PAGETABLE_LEVELS >= 3
|
||||||
#ifdef CONFIG_X86_PAE
|
#ifdef CONFIG_X86_PAE
|
||||||
|
@ -298,12 +327,12 @@ struct pv_mmu_ops {
|
||||||
|
|
||||||
void (*set_pud)(pud_t *pudp, pud_t pudval);
|
void (*set_pud)(pud_t *pudp, pud_t pudval);
|
||||||
|
|
||||||
pmdval_t (*pmd_val)(pmd_t);
|
struct paravirt_callee_save pmd_val;
|
||||||
pmd_t (*make_pmd)(pmdval_t pmd);
|
struct paravirt_callee_save make_pmd;
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
#if PAGETABLE_LEVELS == 4
|
||||||
pudval_t (*pud_val)(pud_t);
|
struct paravirt_callee_save pud_val;
|
||||||
pud_t (*make_pud)(pudval_t pud);
|
struct paravirt_callee_save make_pud;
|
||||||
|
|
||||||
void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
|
void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
|
||||||
#endif /* PAGETABLE_LEVELS == 4 */
|
#endif /* PAGETABLE_LEVELS == 4 */
|
||||||
|
@ -388,6 +417,8 @@ extern struct pv_lock_ops pv_lock_ops;
|
||||||
asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
|
asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
|
||||||
|
|
||||||
unsigned paravirt_patch_nop(void);
|
unsigned paravirt_patch_nop(void);
|
||||||
|
unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
|
||||||
|
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
|
||||||
unsigned paravirt_patch_ignore(unsigned len);
|
unsigned paravirt_patch_ignore(unsigned len);
|
||||||
unsigned paravirt_patch_call(void *insnbuf,
|
unsigned paravirt_patch_call(void *insnbuf,
|
||||||
const void *target, u16 tgt_clobbers,
|
const void *target, u16 tgt_clobbers,
|
||||||
|
@ -479,25 +510,45 @@ int paravirt_disable_iospace(void);
|
||||||
* makes sure the incoming and outgoing types are always correct.
|
* makes sure the incoming and outgoing types are always correct.
|
||||||
*/
|
*/
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
#define PVOP_VCALL_ARGS unsigned long __eax, __edx, __ecx
|
#define PVOP_VCALL_ARGS \
|
||||||
|
unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
|
||||||
#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
|
#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
|
||||||
|
|
||||||
|
#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
|
||||||
|
#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
|
||||||
|
#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
|
||||||
|
|
||||||
#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
|
#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
|
||||||
"=c" (__ecx)
|
"=c" (__ecx)
|
||||||
#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
|
#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
|
||||||
|
|
||||||
|
#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
|
||||||
|
#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
|
||||||
|
|
||||||
#define EXTRA_CLOBBERS
|
#define EXTRA_CLOBBERS
|
||||||
#define VEXTRA_CLOBBERS
|
#define VEXTRA_CLOBBERS
|
||||||
#else
|
#else /* CONFIG_X86_64 */
|
||||||
#define PVOP_VCALL_ARGS unsigned long __edi, __esi, __edx, __ecx
|
#define PVOP_VCALL_ARGS \
|
||||||
|
unsigned long __edi = __edi, __esi = __esi, \
|
||||||
|
__edx = __edx, __ecx = __ecx
|
||||||
#define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax
|
#define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax
|
||||||
|
|
||||||
|
#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
|
||||||
|
#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
|
||||||
|
#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
|
||||||
|
#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
|
||||||
|
|
||||||
#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
|
#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
|
||||||
"=S" (__esi), "=d" (__edx), \
|
"=S" (__esi), "=d" (__edx), \
|
||||||
"=c" (__ecx)
|
"=c" (__ecx)
|
||||||
|
|
||||||
#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
|
#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
|
||||||
|
|
||||||
|
#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
|
||||||
|
#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
|
||||||
|
|
||||||
#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
|
#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
|
||||||
#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
|
#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
|
||||||
#endif
|
#endif /* CONFIG_X86_32 */
|
||||||
|
|
||||||
#ifdef CONFIG_PARAVIRT_DEBUG
|
#ifdef CONFIG_PARAVIRT_DEBUG
|
||||||
#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
|
#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
|
||||||
|
@ -505,10 +556,11 @@ int paravirt_disable_iospace(void);
|
||||||
#define PVOP_TEST_NULL(op) ((void)op)
|
#define PVOP_TEST_NULL(op) ((void)op)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define __PVOP_CALL(rettype, op, pre, post, ...) \
|
#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
|
||||||
|
pre, post, ...) \
|
||||||
({ \
|
({ \
|
||||||
rettype __ret; \
|
rettype __ret; \
|
||||||
PVOP_CALL_ARGS; \
|
PVOP_CALL_ARGS; \
|
||||||
PVOP_TEST_NULL(op); \
|
PVOP_TEST_NULL(op); \
|
||||||
/* This is 32-bit specific, but is okay in 64-bit */ \
|
/* This is 32-bit specific, but is okay in 64-bit */ \
|
||||||
/* since this condition will never hold */ \
|
/* since this condition will never hold */ \
|
||||||
|
@ -516,70 +568,113 @@ int paravirt_disable_iospace(void);
|
||||||
asm volatile(pre \
|
asm volatile(pre \
|
||||||
paravirt_alt(PARAVIRT_CALL) \
|
paravirt_alt(PARAVIRT_CALL) \
|
||||||
post \
|
post \
|
||||||
: PVOP_CALL_CLOBBERS \
|
: call_clbr \
|
||||||
: paravirt_type(op), \
|
: paravirt_type(op), \
|
||||||
paravirt_clobber(CLBR_ANY), \
|
paravirt_clobber(clbr), \
|
||||||
##__VA_ARGS__ \
|
##__VA_ARGS__ \
|
||||||
: "memory", "cc" EXTRA_CLOBBERS); \
|
: "memory", "cc" extra_clbr); \
|
||||||
__ret = (rettype)((((u64)__edx) << 32) | __eax); \
|
__ret = (rettype)((((u64)__edx) << 32) | __eax); \
|
||||||
} else { \
|
} else { \
|
||||||
asm volatile(pre \
|
asm volatile(pre \
|
||||||
paravirt_alt(PARAVIRT_CALL) \
|
paravirt_alt(PARAVIRT_CALL) \
|
||||||
post \
|
post \
|
||||||
: PVOP_CALL_CLOBBERS \
|
: call_clbr \
|
||||||
: paravirt_type(op), \
|
: paravirt_type(op), \
|
||||||
paravirt_clobber(CLBR_ANY), \
|
paravirt_clobber(clbr), \
|
||||||
##__VA_ARGS__ \
|
##__VA_ARGS__ \
|
||||||
: "memory", "cc" EXTRA_CLOBBERS); \
|
: "memory", "cc" extra_clbr); \
|
||||||
__ret = (rettype)__eax; \
|
__ret = (rettype)__eax; \
|
||||||
} \
|
} \
|
||||||
__ret; \
|
__ret; \
|
||||||
})
|
})
|
||||||
#define __PVOP_VCALL(op, pre, post, ...) \
|
|
||||||
|
#define __PVOP_CALL(rettype, op, pre, post, ...) \
|
||||||
|
____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
|
||||||
|
EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
|
||||||
|
|
||||||
|
#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
|
||||||
|
____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
|
||||||
|
PVOP_CALLEE_CLOBBERS, , \
|
||||||
|
pre, post, ##__VA_ARGS__)
|
||||||
|
|
||||||
|
|
||||||
|
#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
|
||||||
({ \
|
({ \
|
||||||
PVOP_VCALL_ARGS; \
|
PVOP_VCALL_ARGS; \
|
||||||
PVOP_TEST_NULL(op); \
|
PVOP_TEST_NULL(op); \
|
||||||
asm volatile(pre \
|
asm volatile(pre \
|
||||||
paravirt_alt(PARAVIRT_CALL) \
|
paravirt_alt(PARAVIRT_CALL) \
|
||||||
post \
|
post \
|
||||||
: PVOP_VCALL_CLOBBERS \
|
: call_clbr \
|
||||||
: paravirt_type(op), \
|
: paravirt_type(op), \
|
||||||
paravirt_clobber(CLBR_ANY), \
|
paravirt_clobber(clbr), \
|
||||||
##__VA_ARGS__ \
|
##__VA_ARGS__ \
|
||||||
: "memory", "cc" VEXTRA_CLOBBERS); \
|
: "memory", "cc" extra_clbr); \
|
||||||
})
|
})
|
||||||
|
|
||||||
|
#define __PVOP_VCALL(op, pre, post, ...) \
|
||||||
|
____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
|
||||||
|
VEXTRA_CLOBBERS, \
|
||||||
|
pre, post, ##__VA_ARGS__)
|
||||||
|
|
||||||
|
#define __PVOP_VCALLEESAVE(rettype, op, pre, post, ...) \
|
||||||
|
____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
|
||||||
|
PVOP_VCALLEE_CLOBBERS, , \
|
||||||
|
pre, post, ##__VA_ARGS__)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#define PVOP_CALL0(rettype, op) \
|
#define PVOP_CALL0(rettype, op) \
|
||||||
__PVOP_CALL(rettype, op, "", "")
|
__PVOP_CALL(rettype, op, "", "")
|
||||||
#define PVOP_VCALL0(op) \
|
#define PVOP_VCALL0(op) \
|
||||||
__PVOP_VCALL(op, "", "")
|
__PVOP_VCALL(op, "", "")
|
||||||
|
|
||||||
|
#define PVOP_CALLEE0(rettype, op) \
|
||||||
|
__PVOP_CALLEESAVE(rettype, op, "", "")
|
||||||
|
#define PVOP_VCALLEE0(op) \
|
||||||
|
__PVOP_VCALLEESAVE(op, "", "")
|
||||||
|
|
||||||
|
|
||||||
#define PVOP_CALL1(rettype, op, arg1) \
|
#define PVOP_CALL1(rettype, op, arg1) \
|
||||||
__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)))
|
__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
|
||||||
#define PVOP_VCALL1(op, arg1) \
|
#define PVOP_VCALL1(op, arg1) \
|
||||||
__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)))
|
__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
|
||||||
|
|
||||||
|
#define PVOP_CALLEE1(rettype, op, arg1) \
|
||||||
|
__PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
|
||||||
|
#define PVOP_VCALLEE1(op, arg1) \
|
||||||
|
__PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
|
||||||
|
|
||||||
|
|
||||||
#define PVOP_CALL2(rettype, op, arg1, arg2) \
|
#define PVOP_CALL2(rettype, op, arg1, arg2) \
|
||||||
__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
"1" ((unsigned long)(arg2)))
|
PVOP_CALL_ARG2(arg2))
|
||||||
#define PVOP_VCALL2(op, arg1, arg2) \
|
#define PVOP_VCALL2(op, arg1, arg2) \
|
||||||
__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
"1" ((unsigned long)(arg2)))
|
PVOP_CALL_ARG2(arg2))
|
||||||
|
|
||||||
|
#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
|
||||||
|
__PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
|
PVOP_CALL_ARG2(arg2))
|
||||||
|
#define PVOP_VCALLEE2(op, arg1, arg2) \
|
||||||
|
__PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
|
PVOP_CALL_ARG2(arg2))
|
||||||
|
|
||||||
|
|
||||||
#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
|
#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
|
||||||
__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
|
PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
|
||||||
#define PVOP_VCALL3(op, arg1, arg2, arg3) \
|
#define PVOP_VCALL3(op, arg1, arg2, arg3) \
|
||||||
__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
|
||||||
"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
|
PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
|
||||||
|
|
||||||
/* This is the only difference in x86_64. We can make it much simpler */
|
/* This is the only difference in x86_64. We can make it much simpler */
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
|
#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
|
||||||
__PVOP_CALL(rettype, op, \
|
__PVOP_CALL(rettype, op, \
|
||||||
"push %[_arg4];", "lea 4(%%esp),%%esp;", \
|
"push %[_arg4];", "lea 4(%%esp),%%esp;", \
|
||||||
"0" ((u32)(arg1)), "1" ((u32)(arg2)), \
|
PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
|
||||||
"2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
|
PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
|
||||||
#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
|
#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
|
||||||
__PVOP_VCALL(op, \
|
__PVOP_VCALL(op, \
|
||||||
"push %[_arg4];", "lea 4(%%esp),%%esp;", \
|
"push %[_arg4];", "lea 4(%%esp),%%esp;", \
|
||||||
|
@ -587,13 +682,13 @@ int paravirt_disable_iospace(void);
|
||||||
"2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
|
"2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
|
||||||
#else
|
#else
|
||||||
#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
|
#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
|
||||||
__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_CALL(rettype, op, "", "", \
|
||||||
"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
|
PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
|
||||||
"3"((unsigned long)(arg4)))
|
PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
|
||||||
#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
|
#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
|
||||||
__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
|
__PVOP_VCALL(op, "", "", \
|
||||||
"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
|
PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
|
||||||
"3"((unsigned long)(arg4)))
|
PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
static inline int paravirt_enabled(void)
|
static inline int paravirt_enabled(void)
|
||||||
|
@ -1060,13 +1155,13 @@ static inline pte_t __pte(pteval_t val)
|
||||||
pteval_t ret;
|
pteval_t ret;
|
||||||
|
|
||||||
if (sizeof(pteval_t) > sizeof(long))
|
if (sizeof(pteval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pteval_t,
|
ret = PVOP_CALLEE2(pteval_t,
|
||||||
pv_mmu_ops.make_pte,
|
pv_mmu_ops.make_pte,
|
||||||
val, (u64)val >> 32);
|
val, (u64)val >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pteval_t,
|
ret = PVOP_CALLEE1(pteval_t,
|
||||||
pv_mmu_ops.make_pte,
|
pv_mmu_ops.make_pte,
|
||||||
val);
|
val);
|
||||||
|
|
||||||
return (pte_t) { .pte = ret };
|
return (pte_t) { .pte = ret };
|
||||||
}
|
}
|
||||||
|
@ -1076,11 +1171,11 @@ static inline pteval_t pte_val(pte_t pte)
|
||||||
pteval_t ret;
|
pteval_t ret;
|
||||||
|
|
||||||
if (sizeof(pteval_t) > sizeof(long))
|
if (sizeof(pteval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_val,
|
ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
|
||||||
pte.pte, (u64)pte.pte >> 32);
|
pte.pte, (u64)pte.pte >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_val,
|
ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
|
||||||
pte.pte);
|
pte.pte);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -1090,11 +1185,11 @@ static inline pgd_t __pgd(pgdval_t val)
|
||||||
pgdval_t ret;
|
pgdval_t ret;
|
||||||
|
|
||||||
if (sizeof(pgdval_t) > sizeof(long))
|
if (sizeof(pgdval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.make_pgd,
|
ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
|
||||||
val, (u64)val >> 32);
|
val, (u64)val >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.make_pgd,
|
ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
|
||||||
val);
|
val);
|
||||||
|
|
||||||
return (pgd_t) { ret };
|
return (pgd_t) { ret };
|
||||||
}
|
}
|
||||||
|
@ -1104,11 +1199,11 @@ static inline pgdval_t pgd_val(pgd_t pgd)
|
||||||
pgdval_t ret;
|
pgdval_t ret;
|
||||||
|
|
||||||
if (sizeof(pgdval_t) > sizeof(long))
|
if (sizeof(pgdval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.pgd_val,
|
ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
|
||||||
pgd.pgd, (u64)pgd.pgd >> 32);
|
pgd.pgd, (u64)pgd.pgd >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.pgd_val,
|
ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
|
||||||
pgd.pgd);
|
pgd.pgd);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -1172,11 +1267,11 @@ static inline pmd_t __pmd(pmdval_t val)
|
||||||
pmdval_t ret;
|
pmdval_t ret;
|
||||||
|
|
||||||
if (sizeof(pmdval_t) > sizeof(long))
|
if (sizeof(pmdval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd,
|
ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
|
||||||
val, (u64)val >> 32);
|
val, (u64)val >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd,
|
ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
|
||||||
val);
|
val);
|
||||||
|
|
||||||
return (pmd_t) { ret };
|
return (pmd_t) { ret };
|
||||||
}
|
}
|
||||||
|
@ -1186,11 +1281,11 @@ static inline pmdval_t pmd_val(pmd_t pmd)
|
||||||
pmdval_t ret;
|
pmdval_t ret;
|
||||||
|
|
||||||
if (sizeof(pmdval_t) > sizeof(long))
|
if (sizeof(pmdval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val,
|
ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
|
||||||
pmd.pmd, (u64)pmd.pmd >> 32);
|
pmd.pmd, (u64)pmd.pmd >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val,
|
ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
|
||||||
pmd.pmd);
|
pmd.pmd);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -1212,11 +1307,11 @@ static inline pud_t __pud(pudval_t val)
|
||||||
pudval_t ret;
|
pudval_t ret;
|
||||||
|
|
||||||
if (sizeof(pudval_t) > sizeof(long))
|
if (sizeof(pudval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud,
|
ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
|
||||||
val, (u64)val >> 32);
|
val, (u64)val >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud,
|
ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
|
||||||
val);
|
val);
|
||||||
|
|
||||||
return (pud_t) { ret };
|
return (pud_t) { ret };
|
||||||
}
|
}
|
||||||
|
@ -1226,11 +1321,11 @@ static inline pudval_t pud_val(pud_t pud)
|
||||||
pudval_t ret;
|
pudval_t ret;
|
||||||
|
|
||||||
if (sizeof(pudval_t) > sizeof(long))
|
if (sizeof(pudval_t) > sizeof(long))
|
||||||
ret = PVOP_CALL2(pudval_t, pv_mmu_ops.pud_val,
|
ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
|
||||||
pud.pud, (u64)pud.pud >> 32);
|
pud.pud, (u64)pud.pud >> 32);
|
||||||
else
|
else
|
||||||
ret = PVOP_CALL1(pudval_t, pv_mmu_ops.pud_val,
|
ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
|
||||||
pud.pud);
|
pud.pud);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -1371,6 +1466,9 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
|
||||||
}
|
}
|
||||||
|
|
||||||
void _paravirt_nop(void);
|
void _paravirt_nop(void);
|
||||||
|
u32 _paravirt_ident_32(u32);
|
||||||
|
u64 _paravirt_ident_64(u64);
|
||||||
|
|
||||||
#define paravirt_nop ((void *)_paravirt_nop)
|
#define paravirt_nop ((void *)_paravirt_nop)
|
||||||
|
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
|
@ -1420,12 +1518,37 @@ extern struct paravirt_patch_site __parainstructions[],
|
||||||
__parainstructions_end[];
|
__parainstructions_end[];
|
||||||
|
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
#define PV_SAVE_REGS "pushl %%ecx; pushl %%edx;"
|
#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
|
||||||
#define PV_RESTORE_REGS "popl %%edx; popl %%ecx"
|
#define PV_RESTORE_REGS "popl %edx; popl %ecx;"
|
||||||
|
|
||||||
|
/* save and restore all caller-save registers, except return value */
|
||||||
|
#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
|
||||||
|
#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
|
||||||
|
|
||||||
#define PV_FLAGS_ARG "0"
|
#define PV_FLAGS_ARG "0"
|
||||||
#define PV_EXTRA_CLOBBERS
|
#define PV_EXTRA_CLOBBERS
|
||||||
#define PV_VEXTRA_CLOBBERS
|
#define PV_VEXTRA_CLOBBERS
|
||||||
#else
|
#else
|
||||||
|
/* save and restore all caller-save registers, except return value */
|
||||||
|
#define PV_SAVE_ALL_CALLER_REGS \
|
||||||
|
"push %rcx;" \
|
||||||
|
"push %rdx;" \
|
||||||
|
"push %rsi;" \
|
||||||
|
"push %rdi;" \
|
||||||
|
"push %r8;" \
|
||||||
|
"push %r9;" \
|
||||||
|
"push %r10;" \
|
||||||
|
"push %r11;"
|
||||||
|
#define PV_RESTORE_ALL_CALLER_REGS \
|
||||||
|
"pop %r11;" \
|
||||||
|
"pop %r10;" \
|
||||||
|
"pop %r9;" \
|
||||||
|
"pop %r8;" \
|
||||||
|
"pop %rdi;" \
|
||||||
|
"pop %rsi;" \
|
||||||
|
"pop %rdx;" \
|
||||||
|
"pop %rcx;"
|
||||||
|
|
||||||
/* We save some registers, but all of them, that's too much. We clobber all
|
/* We save some registers, but all of them, that's too much. We clobber all
|
||||||
* caller saved registers but the argument parameter */
|
* caller saved registers but the argument parameter */
|
||||||
#define PV_SAVE_REGS "pushq %%rdi;"
|
#define PV_SAVE_REGS "pushq %%rdi;"
|
||||||
|
@ -1435,52 +1558,76 @@ extern struct paravirt_patch_site __parainstructions[],
|
||||||
#define PV_FLAGS_ARG "D"
|
#define PV_FLAGS_ARG "D"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Generate a thunk around a function which saves all caller-save
|
||||||
|
* registers except for the return value. This allows C functions to
|
||||||
|
* be called from assembler code where fewer than normal registers are
|
||||||
|
* available. It may also help code generation around calls from C
|
||||||
|
* code if the common case doesn't use many registers.
|
||||||
|
*
|
||||||
|
* When a callee is wrapped in a thunk, the caller can assume that all
|
||||||
|
* arg regs and all scratch registers are preserved across the
|
||||||
|
* call. The return value in rax/eax will not be saved, even for void
|
||||||
|
* functions.
|
||||||
|
*/
|
||||||
|
#define PV_CALLEE_SAVE_REGS_THUNK(func) \
|
||||||
|
extern typeof(func) __raw_callee_save_##func; \
|
||||||
|
static void *__##func##__ __used = func; \
|
||||||
|
\
|
||||||
|
asm(".pushsection .text;" \
|
||||||
|
"__raw_callee_save_" #func ": " \
|
||||||
|
PV_SAVE_ALL_CALLER_REGS \
|
||||||
|
"call " #func ";" \
|
||||||
|
PV_RESTORE_ALL_CALLER_REGS \
|
||||||
|
"ret;" \
|
||||||
|
".popsection")
|
||||||
|
|
||||||
|
/* Get a reference to a callee-save function */
|
||||||
|
#define PV_CALLEE_SAVE(func) \
|
||||||
|
((struct paravirt_callee_save) { __raw_callee_save_##func })
|
||||||
|
|
||||||
|
/* Promise that "func" already uses the right calling convention */
|
||||||
|
#define __PV_IS_CALLEE_SAVE(func) \
|
||||||
|
((struct paravirt_callee_save) { func })
|
||||||
|
|
||||||
static inline unsigned long __raw_local_save_flags(void)
|
static inline unsigned long __raw_local_save_flags(void)
|
||||||
{
|
{
|
||||||
unsigned long f;
|
unsigned long f;
|
||||||
|
|
||||||
asm volatile(paravirt_alt(PV_SAVE_REGS
|
asm volatile(paravirt_alt(PARAVIRT_CALL)
|
||||||
PARAVIRT_CALL
|
|
||||||
PV_RESTORE_REGS)
|
|
||||||
: "=a"(f)
|
: "=a"(f)
|
||||||
: paravirt_type(pv_irq_ops.save_fl),
|
: paravirt_type(pv_irq_ops.save_fl),
|
||||||
paravirt_clobber(CLBR_EAX)
|
paravirt_clobber(CLBR_EAX)
|
||||||
: "memory", "cc" PV_VEXTRA_CLOBBERS);
|
: "memory", "cc");
|
||||||
return f;
|
return f;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void raw_local_irq_restore(unsigned long f)
|
static inline void raw_local_irq_restore(unsigned long f)
|
||||||
{
|
{
|
||||||
asm volatile(paravirt_alt(PV_SAVE_REGS
|
asm volatile(paravirt_alt(PARAVIRT_CALL)
|
||||||
PARAVIRT_CALL
|
|
||||||
PV_RESTORE_REGS)
|
|
||||||
: "=a"(f)
|
: "=a"(f)
|
||||||
: PV_FLAGS_ARG(f),
|
: PV_FLAGS_ARG(f),
|
||||||
paravirt_type(pv_irq_ops.restore_fl),
|
paravirt_type(pv_irq_ops.restore_fl),
|
||||||
paravirt_clobber(CLBR_EAX)
|
paravirt_clobber(CLBR_EAX)
|
||||||
: "memory", "cc" PV_EXTRA_CLOBBERS);
|
: "memory", "cc");
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void raw_local_irq_disable(void)
|
static inline void raw_local_irq_disable(void)
|
||||||
{
|
{
|
||||||
asm volatile(paravirt_alt(PV_SAVE_REGS
|
asm volatile(paravirt_alt(PARAVIRT_CALL)
|
||||||
PARAVIRT_CALL
|
|
||||||
PV_RESTORE_REGS)
|
|
||||||
:
|
:
|
||||||
: paravirt_type(pv_irq_ops.irq_disable),
|
: paravirt_type(pv_irq_ops.irq_disable),
|
||||||
paravirt_clobber(CLBR_EAX)
|
paravirt_clobber(CLBR_EAX)
|
||||||
: "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
|
: "memory", "eax", "cc");
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void raw_local_irq_enable(void)
|
static inline void raw_local_irq_enable(void)
|
||||||
{
|
{
|
||||||
asm volatile(paravirt_alt(PV_SAVE_REGS
|
asm volatile(paravirt_alt(PARAVIRT_CALL)
|
||||||
PARAVIRT_CALL
|
|
||||||
PV_RESTORE_REGS)
|
|
||||||
:
|
:
|
||||||
: paravirt_type(pv_irq_ops.irq_enable),
|
: paravirt_type(pv_irq_ops.irq_enable),
|
||||||
paravirt_clobber(CLBR_EAX)
|
paravirt_clobber(CLBR_EAX)
|
||||||
: "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
|
: "memory", "eax", "cc");
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline unsigned long __raw_local_irq_save(void)
|
static inline unsigned long __raw_local_irq_save(void)
|
||||||
|
@ -1523,33 +1670,49 @@ static inline unsigned long __raw_local_irq_save(void)
|
||||||
.popsection
|
.popsection
|
||||||
|
|
||||||
|
|
||||||
|
#define COND_PUSH(set, mask, reg) \
|
||||||
|
.if ((~(set)) & mask); push %reg; .endif
|
||||||
|
#define COND_POP(set, mask, reg) \
|
||||||
|
.if ((~(set)) & mask); pop %reg; .endif
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_64
|
||||||
#define PV_SAVE_REGS \
|
|
||||||
push %rax; \
|
#define PV_SAVE_REGS(set) \
|
||||||
push %rcx; \
|
COND_PUSH(set, CLBR_RAX, rax); \
|
||||||
push %rdx; \
|
COND_PUSH(set, CLBR_RCX, rcx); \
|
||||||
push %rsi; \
|
COND_PUSH(set, CLBR_RDX, rdx); \
|
||||||
push %rdi; \
|
COND_PUSH(set, CLBR_RSI, rsi); \
|
||||||
push %r8; \
|
COND_PUSH(set, CLBR_RDI, rdi); \
|
||||||
push %r9; \
|
COND_PUSH(set, CLBR_R8, r8); \
|
||||||
push %r10; \
|
COND_PUSH(set, CLBR_R9, r9); \
|
||||||
push %r11
|
COND_PUSH(set, CLBR_R10, r10); \
|
||||||
#define PV_RESTORE_REGS \
|
COND_PUSH(set, CLBR_R11, r11)
|
||||||
pop %r11; \
|
#define PV_RESTORE_REGS(set) \
|
||||||
pop %r10; \
|
COND_POP(set, CLBR_R11, r11); \
|
||||||
pop %r9; \
|
COND_POP(set, CLBR_R10, r10); \
|
||||||
pop %r8; \
|
COND_POP(set, CLBR_R9, r9); \
|
||||||
pop %rdi; \
|
COND_POP(set, CLBR_R8, r8); \
|
||||||
pop %rsi; \
|
COND_POP(set, CLBR_RDI, rdi); \
|
||||||
pop %rdx; \
|
COND_POP(set, CLBR_RSI, rsi); \
|
||||||
pop %rcx; \
|
COND_POP(set, CLBR_RDX, rdx); \
|
||||||
pop %rax
|
COND_POP(set, CLBR_RCX, rcx); \
|
||||||
|
COND_POP(set, CLBR_RAX, rax)
|
||||||
|
|
||||||
#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
|
#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
|
||||||
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
|
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
|
||||||
#define PARA_INDIRECT(addr) *addr(%rip)
|
#define PARA_INDIRECT(addr) *addr(%rip)
|
||||||
#else
|
#else
|
||||||
#define PV_SAVE_REGS pushl %eax; pushl %edi; pushl %ecx; pushl %edx
|
#define PV_SAVE_REGS(set) \
|
||||||
#define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
|
COND_PUSH(set, CLBR_EAX, eax); \
|
||||||
|
COND_PUSH(set, CLBR_EDI, edi); \
|
||||||
|
COND_PUSH(set, CLBR_ECX, ecx); \
|
||||||
|
COND_PUSH(set, CLBR_EDX, edx)
|
||||||
|
#define PV_RESTORE_REGS(set) \
|
||||||
|
COND_POP(set, CLBR_EDX, edx); \
|
||||||
|
COND_POP(set, CLBR_ECX, ecx); \
|
||||||
|
COND_POP(set, CLBR_EDI, edi); \
|
||||||
|
COND_POP(set, CLBR_EAX, eax)
|
||||||
|
|
||||||
#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
|
#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
|
||||||
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
|
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
|
||||||
#define PARA_INDIRECT(addr) *%cs:addr
|
#define PARA_INDIRECT(addr) *%cs:addr
|
||||||
|
@ -1561,15 +1724,15 @@ static inline unsigned long __raw_local_irq_save(void)
|
||||||
|
|
||||||
#define DISABLE_INTERRUPTS(clobbers) \
|
#define DISABLE_INTERRUPTS(clobbers) \
|
||||||
PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
|
PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
|
||||||
PV_SAVE_REGS; \
|
PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
|
||||||
call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
|
call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
|
||||||
PV_RESTORE_REGS;) \
|
PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
|
||||||
|
|
||||||
#define ENABLE_INTERRUPTS(clobbers) \
|
#define ENABLE_INTERRUPTS(clobbers) \
|
||||||
PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
|
PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
|
||||||
PV_SAVE_REGS; \
|
PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
|
||||||
call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
|
call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
|
||||||
PV_RESTORE_REGS;)
|
PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
|
||||||
|
|
||||||
#define USERGS_SYSRET32 \
|
#define USERGS_SYSRET32 \
|
||||||
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
|
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
|
||||||
|
@ -1599,11 +1762,15 @@ static inline unsigned long __raw_local_irq_save(void)
|
||||||
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
|
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
|
||||||
swapgs)
|
swapgs)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Note: swapgs is very special, and in practise is either going to be
|
||||||
|
* implemented with a single "swapgs" instruction or something very
|
||||||
|
* special. Either way, we don't need to save any registers for
|
||||||
|
* it.
|
||||||
|
*/
|
||||||
#define SWAPGS \
|
#define SWAPGS \
|
||||||
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
|
PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
|
||||||
PV_SAVE_REGS; \
|
call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
|
||||||
call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs); \
|
|
||||||
PV_RESTORE_REGS \
|
|
||||||
)
|
)
|
||||||
|
|
||||||
#define GET_CR2_INTO_RCX \
|
#define GET_CR2_INTO_RCX \
|
||||||
|
|
|
@ -768,7 +768,8 @@ extern int sysenter_setup(void);
|
||||||
extern struct desc_ptr early_gdt_descr;
|
extern struct desc_ptr early_gdt_descr;
|
||||||
|
|
||||||
extern void cpu_set_gdt(int);
|
extern void cpu_set_gdt(int);
|
||||||
extern void switch_to_new_gdt(void);
|
extern void switch_to_new_gdt(int);
|
||||||
|
extern void load_percpu_segment(int);
|
||||||
extern void cpu_init(void);
|
extern void cpu_init(void);
|
||||||
|
|
||||||
static inline unsigned long get_debugctlmsr(void)
|
static inline unsigned long get_debugctlmsr(void)
|
||||||
|
|
|
@ -296,17 +296,8 @@ static char __cpuinit *table_lookup_model(struct cpuinfo_x86 *c)
|
||||||
|
|
||||||
__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
|
__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;
|
||||||
|
|
||||||
/* Current gdt points %fs at the "master" per-cpu area: after this,
|
void load_percpu_segment(int cpu)
|
||||||
* it's on the real one. */
|
|
||||||
void switch_to_new_gdt(void)
|
|
||||||
{
|
{
|
||||||
struct desc_ptr gdt_descr;
|
|
||||||
int cpu = smp_processor_id();
|
|
||||||
|
|
||||||
gdt_descr.address = (long)get_cpu_gdt_table(cpu);
|
|
||||||
gdt_descr.size = GDT_SIZE - 1;
|
|
||||||
load_gdt(&gdt_descr);
|
|
||||||
/* Reload the per-cpu base */
|
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
loadsegment(fs, __KERNEL_PERCPU);
|
loadsegment(fs, __KERNEL_PERCPU);
|
||||||
#else
|
#else
|
||||||
|
@ -315,6 +306,20 @@ void switch_to_new_gdt(void)
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Current gdt points %fs at the "master" per-cpu area: after this,
|
||||||
|
* it's on the real one. */
|
||||||
|
void switch_to_new_gdt(int cpu)
|
||||||
|
{
|
||||||
|
struct desc_ptr gdt_descr;
|
||||||
|
|
||||||
|
gdt_descr.address = (long)get_cpu_gdt_table(cpu);
|
||||||
|
gdt_descr.size = GDT_SIZE - 1;
|
||||||
|
load_gdt(&gdt_descr);
|
||||||
|
/* Reload the per-cpu base */
|
||||||
|
|
||||||
|
load_percpu_segment(cpu);
|
||||||
|
}
|
||||||
|
|
||||||
static struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
|
static struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
|
||||||
|
|
||||||
static void __cpuinit default_init(struct cpuinfo_x86 *c)
|
static void __cpuinit default_init(struct cpuinfo_x86 *c)
|
||||||
|
@ -1029,7 +1034,7 @@ void __cpuinit cpu_init(void)
|
||||||
* and set up the GDT descriptor:
|
* and set up the GDT descriptor:
|
||||||
*/
|
*/
|
||||||
|
|
||||||
switch_to_new_gdt();
|
switch_to_new_gdt(cpu);
|
||||||
loadsegment(fs, 0);
|
loadsegment(fs, 0);
|
||||||
|
|
||||||
load_idt((const struct desc_ptr *)&idt_descr);
|
load_idt((const struct desc_ptr *)&idt_descr);
|
||||||
|
@ -1131,7 +1136,7 @@ void __cpuinit cpu_init(void)
|
||||||
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
|
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
|
||||||
|
|
||||||
load_idt(&idt_descr);
|
load_idt(&idt_descr);
|
||||||
switch_to_new_gdt();
|
switch_to_new_gdt(cpu);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Set up and load the per-CPU TSS and LDT
|
* Set up and load the per-CPU TSS and LDT
|
||||||
|
|
|
@ -1143,7 +1143,7 @@ ENTRY(native_load_gs_index)
|
||||||
CFI_STARTPROC
|
CFI_STARTPROC
|
||||||
pushf
|
pushf
|
||||||
CFI_ADJUST_CFA_OFFSET 8
|
CFI_ADJUST_CFA_OFFSET 8
|
||||||
DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
|
DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
|
||||||
SWAPGS
|
SWAPGS
|
||||||
gs_change:
|
gs_change:
|
||||||
movl %edi,%gs
|
movl %edi,%gs
|
||||||
|
|
|
@ -44,6 +44,17 @@ void _paravirt_nop(void)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* identity function, which can be inlined */
|
||||||
|
u32 _paravirt_ident_32(u32 x)
|
||||||
|
{
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
u64 _paravirt_ident_64(u64 x)
|
||||||
|
{
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
|
||||||
static void __init default_banner(void)
|
static void __init default_banner(void)
|
||||||
{
|
{
|
||||||
printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
|
printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
|
||||||
|
@ -138,9 +149,16 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
|
||||||
if (opfunc == NULL)
|
if (opfunc == NULL)
|
||||||
/* If there's no function, patch it with a ud2a (BUG) */
|
/* If there's no function, patch it with a ud2a (BUG) */
|
||||||
ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
|
ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
|
||||||
else if (opfunc == paravirt_nop)
|
else if (opfunc == _paravirt_nop)
|
||||||
/* If the operation is a nop, then nop the callsite */
|
/* If the operation is a nop, then nop the callsite */
|
||||||
ret = paravirt_patch_nop();
|
ret = paravirt_patch_nop();
|
||||||
|
|
||||||
|
/* identity functions just return their single argument */
|
||||||
|
else if (opfunc == _paravirt_ident_32)
|
||||||
|
ret = paravirt_patch_ident_32(insnbuf, len);
|
||||||
|
else if (opfunc == _paravirt_ident_64)
|
||||||
|
ret = paravirt_patch_ident_64(insnbuf, len);
|
||||||
|
|
||||||
else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
|
else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
|
||||||
type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
|
type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
|
||||||
type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
|
type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
|
||||||
|
@ -292,10 +310,10 @@ struct pv_time_ops pv_time_ops = {
|
||||||
|
|
||||||
struct pv_irq_ops pv_irq_ops = {
|
struct pv_irq_ops pv_irq_ops = {
|
||||||
.init_IRQ = native_init_IRQ,
|
.init_IRQ = native_init_IRQ,
|
||||||
.save_fl = native_save_fl,
|
.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
|
||||||
.restore_fl = native_restore_fl,
|
.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
|
||||||
.irq_disable = native_irq_disable,
|
.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
|
||||||
.irq_enable = native_irq_enable,
|
.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
|
||||||
.safe_halt = native_safe_halt,
|
.safe_halt = native_safe_halt,
|
||||||
.halt = native_halt,
|
.halt = native_halt,
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_64
|
||||||
|
@ -373,6 +391,14 @@ struct pv_apic_ops pv_apic_ops = {
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
|
||||||
|
/* 32-bit pagetable entries */
|
||||||
|
#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
|
||||||
|
#else
|
||||||
|
/* 64-bit pagetable entries */
|
||||||
|
#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
|
||||||
|
#endif
|
||||||
|
|
||||||
struct pv_mmu_ops pv_mmu_ops = {
|
struct pv_mmu_ops pv_mmu_ops = {
|
||||||
#ifndef CONFIG_X86_64
|
#ifndef CONFIG_X86_64
|
||||||
.pagetable_setup_start = native_pagetable_setup_start,
|
.pagetable_setup_start = native_pagetable_setup_start,
|
||||||
|
@ -424,21 +450,23 @@ struct pv_mmu_ops pv_mmu_ops = {
|
||||||
.pmd_clear = native_pmd_clear,
|
.pmd_clear = native_pmd_clear,
|
||||||
#endif
|
#endif
|
||||||
.set_pud = native_set_pud,
|
.set_pud = native_set_pud,
|
||||||
.pmd_val = native_pmd_val,
|
|
||||||
.make_pmd = native_make_pmd,
|
.pmd_val = PTE_IDENT,
|
||||||
|
.make_pmd = PTE_IDENT,
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
#if PAGETABLE_LEVELS == 4
|
||||||
.pud_val = native_pud_val,
|
.pud_val = PTE_IDENT,
|
||||||
.make_pud = native_make_pud,
|
.make_pud = PTE_IDENT,
|
||||||
|
|
||||||
.set_pgd = native_set_pgd,
|
.set_pgd = native_set_pgd,
|
||||||
#endif
|
#endif
|
||||||
#endif /* PAGETABLE_LEVELS >= 3 */
|
#endif /* PAGETABLE_LEVELS >= 3 */
|
||||||
|
|
||||||
.pte_val = native_pte_val,
|
.pte_val = PTE_IDENT,
|
||||||
.pgd_val = native_pgd_val,
|
.pgd_val = PTE_IDENT,
|
||||||
|
|
||||||
.make_pte = native_make_pte,
|
.make_pte = PTE_IDENT,
|
||||||
.make_pgd = native_make_pgd,
|
.make_pgd = PTE_IDENT,
|
||||||
|
|
||||||
.dup_mmap = paravirt_nop,
|
.dup_mmap = paravirt_nop,
|
||||||
.exit_mmap = paravirt_nop,
|
.exit_mmap = paravirt_nop,
|
||||||
|
|
|
@ -12,6 +12,18 @@ DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");
|
||||||
DEF_NATIVE(pv_cpu_ops, clts, "clts");
|
DEF_NATIVE(pv_cpu_ops, clts, "clts");
|
||||||
DEF_NATIVE(pv_cpu_ops, read_tsc, "rdtsc");
|
DEF_NATIVE(pv_cpu_ops, read_tsc, "rdtsc");
|
||||||
|
|
||||||
|
unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
|
||||||
|
{
|
||||||
|
/* arg in %eax, return in %eax */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
|
||||||
|
{
|
||||||
|
/* arg in %edx:%eax, return in %edx:%eax */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
|
unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
|
||||||
unsigned long addr, unsigned len)
|
unsigned long addr, unsigned len)
|
||||||
{
|
{
|
||||||
|
|
|
@ -19,6 +19,21 @@ DEF_NATIVE(pv_cpu_ops, usergs_sysret64, "swapgs; sysretq");
|
||||||
DEF_NATIVE(pv_cpu_ops, usergs_sysret32, "swapgs; sysretl");
|
DEF_NATIVE(pv_cpu_ops, usergs_sysret32, "swapgs; sysretl");
|
||||||
DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs");
|
DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs");
|
||||||
|
|
||||||
|
DEF_NATIVE(, mov32, "mov %edi, %eax");
|
||||||
|
DEF_NATIVE(, mov64, "mov %rdi, %rax");
|
||||||
|
|
||||||
|
unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
|
||||||
|
{
|
||||||
|
return paravirt_patch_insns(insnbuf, len,
|
||||||
|
start__mov32, end__mov32);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
|
||||||
|
{
|
||||||
|
return paravirt_patch_insns(insnbuf, len,
|
||||||
|
start__mov64, end__mov64);
|
||||||
|
}
|
||||||
|
|
||||||
unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
|
unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
|
||||||
unsigned long addr, unsigned len)
|
unsigned long addr, unsigned len)
|
||||||
{
|
{
|
||||||
|
|
|
@ -122,7 +122,7 @@ void __init setup_per_cpu_areas(void)
|
||||||
* area. Reload any changed state for the boot CPU.
|
* area. Reload any changed state for the boot CPU.
|
||||||
*/
|
*/
|
||||||
if (cpu == boot_cpu_id)
|
if (cpu == boot_cpu_id)
|
||||||
switch_to_new_gdt();
|
switch_to_new_gdt(cpu);
|
||||||
|
|
||||||
DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
|
DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1188,7 +1188,7 @@ out:
|
||||||
void __init native_smp_prepare_boot_cpu(void)
|
void __init native_smp_prepare_boot_cpu(void)
|
||||||
{
|
{
|
||||||
int me = smp_processor_id();
|
int me = smp_processor_id();
|
||||||
switch_to_new_gdt();
|
switch_to_new_gdt(me);
|
||||||
/* already set me in cpu_online_mask in boot_cpu_init() */
|
/* already set me in cpu_online_mask in boot_cpu_init() */
|
||||||
cpumask_set_cpu(me, cpu_callout_mask);
|
cpumask_set_cpu(me, cpu_callout_mask);
|
||||||
per_cpu(cpu_state, me) = CPU_ONLINE;
|
per_cpu(cpu_state, me) = CPU_ONLINE;
|
||||||
|
|
|
@ -259,7 +259,7 @@ const struct cpumask *uv_flush_send_and_wait(int cpu, int this_blade,
|
||||||
* the cpu's, all of which are still in the mask.
|
* the cpu's, all of which are still in the mask.
|
||||||
*/
|
*/
|
||||||
__get_cpu_var(ptcstats).ptc_i++;
|
__get_cpu_var(ptcstats).ptc_i++;
|
||||||
return 0;
|
return flush_mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -670,10 +670,11 @@ static inline int __init activate_vmi(void)
|
||||||
para_fill(pv_mmu_ops.write_cr2, SetCR2);
|
para_fill(pv_mmu_ops.write_cr2, SetCR2);
|
||||||
para_fill(pv_mmu_ops.write_cr3, SetCR3);
|
para_fill(pv_mmu_ops.write_cr3, SetCR3);
|
||||||
para_fill(pv_cpu_ops.write_cr4, SetCR4);
|
para_fill(pv_cpu_ops.write_cr4, SetCR4);
|
||||||
para_fill(pv_irq_ops.save_fl, GetInterruptMask);
|
|
||||||
para_fill(pv_irq_ops.restore_fl, SetInterruptMask);
|
para_fill(pv_irq_ops.save_fl.func, GetInterruptMask);
|
||||||
para_fill(pv_irq_ops.irq_disable, DisableInterrupts);
|
para_fill(pv_irq_ops.restore_fl.func, SetInterruptMask);
|
||||||
para_fill(pv_irq_ops.irq_enable, EnableInterrupts);
|
para_fill(pv_irq_ops.irq_disable.func, DisableInterrupts);
|
||||||
|
para_fill(pv_irq_ops.irq_enable.func, EnableInterrupts);
|
||||||
|
|
||||||
para_fill(pv_cpu_ops.wbinvd, WBINVD);
|
para_fill(pv_cpu_ops.wbinvd, WBINVD);
|
||||||
para_fill(pv_cpu_ops.read_tsc, RDTSC);
|
para_fill(pv_cpu_ops.read_tsc, RDTSC);
|
||||||
|
|
|
@ -22,6 +22,7 @@ PHDRS {
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_SMP
|
||||||
percpu PT_LOAD FLAGS(7); /* RWE */
|
percpu PT_LOAD FLAGS(7); /* RWE */
|
||||||
#endif
|
#endif
|
||||||
|
data.init2 PT_LOAD FLAGS(7); /* RWE */
|
||||||
note PT_NOTE FLAGS(0); /* ___ */
|
note PT_NOTE FLAGS(0); /* ___ */
|
||||||
}
|
}
|
||||||
SECTIONS
|
SECTIONS
|
||||||
|
@ -215,7 +216,7 @@ SECTIONS
|
||||||
/*
|
/*
|
||||||
* percpu offsets are zero-based on SMP. PERCPU_VADDR() changes the
|
* percpu offsets are zero-based on SMP. PERCPU_VADDR() changes the
|
||||||
* output PHDR, so the next output section - __data_nosave - should
|
* output PHDR, so the next output section - __data_nosave - should
|
||||||
* switch it back to data.init. Also, pda should be at the head of
|
* start another section data.init2. Also, pda should be at the head of
|
||||||
* percpu area. Preallocate it and define the percpu offset symbol
|
* percpu area. Preallocate it and define the percpu offset symbol
|
||||||
* so that it can be accessed as a percpu variable.
|
* so that it can be accessed as a percpu variable.
|
||||||
*/
|
*/
|
||||||
|
@ -232,7 +233,7 @@ SECTIONS
|
||||||
__nosave_begin = .;
|
__nosave_begin = .;
|
||||||
.data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
|
.data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
|
||||||
*(.data.nosave)
|
*(.data.nosave)
|
||||||
} :data.init /* switch back to data.init, see PERCPU_VADDR() above */
|
} :data.init2 /* use another section data.init2, see PERCPU_VADDR() above */
|
||||||
. = ALIGN(PAGE_SIZE);
|
. = ALIGN(PAGE_SIZE);
|
||||||
__nosave_end = .;
|
__nosave_end = .;
|
||||||
|
|
||||||
|
|
|
@ -37,6 +37,7 @@ static unsigned long vsmp_save_fl(void)
|
||||||
flags &= ~X86_EFLAGS_IF;
|
flags &= ~X86_EFLAGS_IF;
|
||||||
return flags;
|
return flags;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(vsmp_save_fl);
|
||||||
|
|
||||||
static void vsmp_restore_fl(unsigned long flags)
|
static void vsmp_restore_fl(unsigned long flags)
|
||||||
{
|
{
|
||||||
|
@ -46,6 +47,7 @@ static void vsmp_restore_fl(unsigned long flags)
|
||||||
flags |= X86_EFLAGS_AC;
|
flags |= X86_EFLAGS_AC;
|
||||||
native_restore_fl(flags);
|
native_restore_fl(flags);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(vsmp_restore_fl);
|
||||||
|
|
||||||
static void vsmp_irq_disable(void)
|
static void vsmp_irq_disable(void)
|
||||||
{
|
{
|
||||||
|
@ -53,6 +55,7 @@ static void vsmp_irq_disable(void)
|
||||||
|
|
||||||
native_restore_fl((flags & ~X86_EFLAGS_IF) | X86_EFLAGS_AC);
|
native_restore_fl((flags & ~X86_EFLAGS_IF) | X86_EFLAGS_AC);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(vsmp_irq_disable);
|
||||||
|
|
||||||
static void vsmp_irq_enable(void)
|
static void vsmp_irq_enable(void)
|
||||||
{
|
{
|
||||||
|
@ -60,6 +63,7 @@ static void vsmp_irq_enable(void)
|
||||||
|
|
||||||
native_restore_fl((flags | X86_EFLAGS_IF) & (~X86_EFLAGS_AC));
|
native_restore_fl((flags | X86_EFLAGS_IF) & (~X86_EFLAGS_AC));
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(vsmp_irq_enable);
|
||||||
|
|
||||||
static unsigned __init_or_module vsmp_patch(u8 type, u16 clobbers, void *ibuf,
|
static unsigned __init_or_module vsmp_patch(u8 type, u16 clobbers, void *ibuf,
|
||||||
unsigned long addr, unsigned len)
|
unsigned long addr, unsigned len)
|
||||||
|
@ -90,10 +94,10 @@ static void __init set_vsmp_pv_ops(void)
|
||||||
cap, ctl);
|
cap, ctl);
|
||||||
if (cap & ctl & (1 << 4)) {
|
if (cap & ctl & (1 << 4)) {
|
||||||
/* Setup irq ops and turn on vSMP IRQ fastpath handling */
|
/* Setup irq ops and turn on vSMP IRQ fastpath handling */
|
||||||
pv_irq_ops.irq_disable = vsmp_irq_disable;
|
pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
|
||||||
pv_irq_ops.irq_enable = vsmp_irq_enable;
|
pv_irq_ops.irq_enable = PV_CALLEE_SAVE(vsmp_irq_enable);
|
||||||
pv_irq_ops.save_fl = vsmp_save_fl;
|
pv_irq_ops.save_fl = PV_CALLEE_SAVE(vsmp_save_fl);
|
||||||
pv_irq_ops.restore_fl = vsmp_restore_fl;
|
pv_irq_ops.restore_fl = PV_CALLEE_SAVE(vsmp_restore_fl);
|
||||||
pv_init_ops.patch = vsmp_patch;
|
pv_init_ops.patch = vsmp_patch;
|
||||||
|
|
||||||
ctl &= ~(1 << 4);
|
ctl &= ~(1 << 4);
|
||||||
|
|
|
@ -173,24 +173,29 @@ static unsigned long save_fl(void)
|
||||||
{
|
{
|
||||||
return lguest_data.irq_enabled;
|
return lguest_data.irq_enabled;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(save_fl);
|
||||||
|
|
||||||
/* restore_flags() just sets the flags back to the value given. */
|
/* restore_flags() just sets the flags back to the value given. */
|
||||||
static void restore_fl(unsigned long flags)
|
static void restore_fl(unsigned long flags)
|
||||||
{
|
{
|
||||||
lguest_data.irq_enabled = flags;
|
lguest_data.irq_enabled = flags;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(restore_fl);
|
||||||
|
|
||||||
/* Interrupts go off... */
|
/* Interrupts go off... */
|
||||||
static void irq_disable(void)
|
static void irq_disable(void)
|
||||||
{
|
{
|
||||||
lguest_data.irq_enabled = 0;
|
lguest_data.irq_enabled = 0;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(irq_disable);
|
||||||
|
|
||||||
/* Interrupts go on... */
|
/* Interrupts go on... */
|
||||||
static void irq_enable(void)
|
static void irq_enable(void)
|
||||||
{
|
{
|
||||||
lguest_data.irq_enabled = X86_EFLAGS_IF;
|
lguest_data.irq_enabled = X86_EFLAGS_IF;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(irq_enable);
|
||||||
|
|
||||||
/*:*/
|
/*:*/
|
||||||
/*M:003 Note that we don't check for outstanding interrupts when we re-enable
|
/*M:003 Note that we don't check for outstanding interrupts when we re-enable
|
||||||
* them (or when we unmask an interrupt). This seems to work for the moment,
|
* them (or when we unmask an interrupt). This seems to work for the moment,
|
||||||
|
@ -984,10 +989,10 @@ __init void lguest_init(void)
|
||||||
|
|
||||||
/* interrupt-related operations */
|
/* interrupt-related operations */
|
||||||
pv_irq_ops.init_IRQ = lguest_init_IRQ;
|
pv_irq_ops.init_IRQ = lguest_init_IRQ;
|
||||||
pv_irq_ops.save_fl = save_fl;
|
pv_irq_ops.save_fl = PV_CALLEE_SAVE(save_fl);
|
||||||
pv_irq_ops.restore_fl = restore_fl;
|
pv_irq_ops.restore_fl = PV_CALLEE_SAVE(restore_fl);
|
||||||
pv_irq_ops.irq_disable = irq_disable;
|
pv_irq_ops.irq_disable = PV_CALLEE_SAVE(irq_disable);
|
||||||
pv_irq_ops.irq_enable = irq_enable;
|
pv_irq_ops.irq_enable = PV_CALLEE_SAVE(irq_enable);
|
||||||
pv_irq_ops.safe_halt = lguest_safe_halt;
|
pv_irq_ops.safe_halt = lguest_safe_halt;
|
||||||
|
|
||||||
/* init-time operations */
|
/* init-time operations */
|
||||||
|
|
|
@ -1744,13 +1744,13 @@ static void __init voyager_smp_prepare_cpus(unsigned int max_cpus)
|
||||||
|
|
||||||
static void __cpuinit voyager_smp_prepare_boot_cpu(void)
|
static void __cpuinit voyager_smp_prepare_boot_cpu(void)
|
||||||
{
|
{
|
||||||
switch_to_new_gdt();
|
int cpu = smp_processor_id();
|
||||||
|
switch_to_new_gdt(cpu);
|
||||||
|
|
||||||
cpu_online_map = cpumask_of_cpu(smp_processor_id());
|
cpu_online_map = cpumask_of_cpu(smp_processor_id());
|
||||||
cpu_callout_map = cpumask_of_cpu(smp_processor_id());
|
cpu_callout_map = cpumask_of_cpu(smp_processor_id());
|
||||||
cpu_callin_map = CPU_MASK_NONE;
|
cpu_callin_map = CPU_MASK_NONE;
|
||||||
cpu_present_map = cpumask_of_cpu(smp_processor_id());
|
cpu_present_map = cpumask_of_cpu(smp_processor_id());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static int __cpuinit voyager_cpu_up(unsigned int cpu)
|
static int __cpuinit voyager_cpu_up(unsigned int cpu)
|
||||||
|
|
|
@ -6,7 +6,8 @@ CFLAGS_REMOVE_irq.o = -pg
|
||||||
endif
|
endif
|
||||||
|
|
||||||
obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \
|
obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \
|
||||||
time.o xen-asm_$(BITS).o grant-table.o suspend.o
|
time.o xen-asm.o xen-asm_$(BITS).o \
|
||||||
|
grant-table.o suspend.o
|
||||||
|
|
||||||
obj-$(CONFIG_SMP) += smp.o spinlock.o
|
obj-$(CONFIG_SMP) += smp.o spinlock.o
|
||||||
obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o
|
obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o
|
|
@ -61,40 +61,13 @@ DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
|
||||||
enum xen_domain_type xen_domain_type = XEN_NATIVE;
|
enum xen_domain_type xen_domain_type = XEN_NATIVE;
|
||||||
EXPORT_SYMBOL_GPL(xen_domain_type);
|
EXPORT_SYMBOL_GPL(xen_domain_type);
|
||||||
|
|
||||||
/*
|
|
||||||
* Identity map, in addition to plain kernel map. This needs to be
|
|
||||||
* large enough to allocate page table pages to allocate the rest.
|
|
||||||
* Each page can map 2MB.
|
|
||||||
*/
|
|
||||||
static pte_t level1_ident_pgt[PTRS_PER_PTE * 4] __page_aligned_bss;
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
/* l3 pud for userspace vsyscall mapping */
|
|
||||||
static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
|
|
||||||
#endif /* CONFIG_X86_64 */
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Note about cr3 (pagetable base) values:
|
|
||||||
*
|
|
||||||
* xen_cr3 contains the current logical cr3 value; it contains the
|
|
||||||
* last set cr3. This may not be the current effective cr3, because
|
|
||||||
* its update may be being lazily deferred. However, a vcpu looking
|
|
||||||
* at its own cr3 can use this value knowing that it everything will
|
|
||||||
* be self-consistent.
|
|
||||||
*
|
|
||||||
* xen_current_cr3 contains the actual vcpu cr3; it is set once the
|
|
||||||
* hypercall to set the vcpu cr3 is complete (so it may be a little
|
|
||||||
* out of date, but it will never be set early). If one vcpu is
|
|
||||||
* looking at another vcpu's cr3 value, it should use this variable.
|
|
||||||
*/
|
|
||||||
DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
|
|
||||||
DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
|
|
||||||
|
|
||||||
struct start_info *xen_start_info;
|
struct start_info *xen_start_info;
|
||||||
EXPORT_SYMBOL_GPL(xen_start_info);
|
EXPORT_SYMBOL_GPL(xen_start_info);
|
||||||
|
|
||||||
struct shared_info xen_dummy_shared_info;
|
struct shared_info xen_dummy_shared_info;
|
||||||
|
|
||||||
|
void *xen_initial_gdt;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Point at some empty memory to start with. We map the real shared_info
|
* Point at some empty memory to start with. We map the real shared_info
|
||||||
* page as soon as fixmap is up and running.
|
* page as soon as fixmap is up and running.
|
||||||
|
@ -114,14 +87,7 @@ struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
|
||||||
*
|
*
|
||||||
* 0: not available, 1: available
|
* 0: not available, 1: available
|
||||||
*/
|
*/
|
||||||
static int have_vcpu_info_placement =
|
static int have_vcpu_info_placement = 1;
|
||||||
#ifdef CONFIG_X86_32
|
|
||||||
1
|
|
||||||
#else
|
|
||||||
0
|
|
||||||
#endif
|
|
||||||
;
|
|
||||||
|
|
||||||
|
|
||||||
static void xen_vcpu_setup(int cpu)
|
static void xen_vcpu_setup(int cpu)
|
||||||
{
|
{
|
||||||
|
@ -237,7 +203,7 @@ static unsigned long xen_get_debugreg(int reg)
|
||||||
return HYPERVISOR_get_debugreg(reg);
|
return HYPERVISOR_get_debugreg(reg);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void xen_leave_lazy(void)
|
void xen_leave_lazy(void)
|
||||||
{
|
{
|
||||||
paravirt_leave_lazy(paravirt_get_lazy_mode());
|
paravirt_leave_lazy(paravirt_get_lazy_mode());
|
||||||
xen_mc_flush();
|
xen_mc_flush();
|
||||||
|
@ -598,76 +564,6 @@ static struct apic_ops xen_basic_apic_ops = {
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
static void xen_flush_tlb(void)
|
|
||||||
{
|
|
||||||
struct mmuext_op *op;
|
|
||||||
struct multicall_space mcs;
|
|
||||||
|
|
||||||
preempt_disable();
|
|
||||||
|
|
||||||
mcs = xen_mc_entry(sizeof(*op));
|
|
||||||
|
|
||||||
op = mcs.args;
|
|
||||||
op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
|
|
||||||
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
|
||||||
|
|
||||||
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
|
||||||
|
|
||||||
preempt_enable();
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_flush_tlb_single(unsigned long addr)
|
|
||||||
{
|
|
||||||
struct mmuext_op *op;
|
|
||||||
struct multicall_space mcs;
|
|
||||||
|
|
||||||
preempt_disable();
|
|
||||||
|
|
||||||
mcs = xen_mc_entry(sizeof(*op));
|
|
||||||
op = mcs.args;
|
|
||||||
op->cmd = MMUEXT_INVLPG_LOCAL;
|
|
||||||
op->arg1.linear_addr = addr & PAGE_MASK;
|
|
||||||
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
|
||||||
|
|
||||||
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
|
||||||
|
|
||||||
preempt_enable();
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_flush_tlb_others(const struct cpumask *cpus,
|
|
||||||
struct mm_struct *mm, unsigned long va)
|
|
||||||
{
|
|
||||||
struct {
|
|
||||||
struct mmuext_op op;
|
|
||||||
DECLARE_BITMAP(mask, NR_CPUS);
|
|
||||||
} *args;
|
|
||||||
struct multicall_space mcs;
|
|
||||||
|
|
||||||
BUG_ON(cpumask_empty(cpus));
|
|
||||||
BUG_ON(!mm);
|
|
||||||
|
|
||||||
mcs = xen_mc_entry(sizeof(*args));
|
|
||||||
args = mcs.args;
|
|
||||||
args->op.arg2.vcpumask = to_cpumask(args->mask);
|
|
||||||
|
|
||||||
/* Remove us, and any offline CPUS. */
|
|
||||||
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
|
|
||||||
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
|
|
||||||
if (unlikely(cpumask_empty(to_cpumask(args->mask))))
|
|
||||||
goto issue;
|
|
||||||
|
|
||||||
if (va == TLB_FLUSH_ALL) {
|
|
||||||
args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
|
|
||||||
} else {
|
|
||||||
args->op.cmd = MMUEXT_INVLPG_MULTI;
|
|
||||||
args->op.arg1.linear_addr = va;
|
|
||||||
}
|
|
||||||
|
|
||||||
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
|
|
||||||
|
|
||||||
issue:
|
|
||||||
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_clts(void)
|
static void xen_clts(void)
|
||||||
{
|
{
|
||||||
|
@ -693,21 +589,6 @@ static void xen_write_cr0(unsigned long cr0)
|
||||||
xen_mc_issue(PARAVIRT_LAZY_CPU);
|
xen_mc_issue(PARAVIRT_LAZY_CPU);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void xen_write_cr2(unsigned long cr2)
|
|
||||||
{
|
|
||||||
percpu_read(xen_vcpu)->arch.cr2 = cr2;
|
|
||||||
}
|
|
||||||
|
|
||||||
static unsigned long xen_read_cr2(void)
|
|
||||||
{
|
|
||||||
return percpu_read(xen_vcpu)->arch.cr2;
|
|
||||||
}
|
|
||||||
|
|
||||||
static unsigned long xen_read_cr2_direct(void)
|
|
||||||
{
|
|
||||||
return percpu_read(xen_vcpu_info.arch.cr2);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_write_cr4(unsigned long cr4)
|
static void xen_write_cr4(unsigned long cr4)
|
||||||
{
|
{
|
||||||
cr4 &= ~X86_CR4_PGE;
|
cr4 &= ~X86_CR4_PGE;
|
||||||
|
@ -716,71 +597,6 @@ static void xen_write_cr4(unsigned long cr4)
|
||||||
native_write_cr4(cr4);
|
native_write_cr4(cr4);
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned long xen_read_cr3(void)
|
|
||||||
{
|
|
||||||
return percpu_read(xen_cr3);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void set_current_cr3(void *v)
|
|
||||||
{
|
|
||||||
percpu_write(xen_current_cr3, (unsigned long)v);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void __xen_write_cr3(bool kernel, unsigned long cr3)
|
|
||||||
{
|
|
||||||
struct mmuext_op *op;
|
|
||||||
struct multicall_space mcs;
|
|
||||||
unsigned long mfn;
|
|
||||||
|
|
||||||
if (cr3)
|
|
||||||
mfn = pfn_to_mfn(PFN_DOWN(cr3));
|
|
||||||
else
|
|
||||||
mfn = 0;
|
|
||||||
|
|
||||||
WARN_ON(mfn == 0 && kernel);
|
|
||||||
|
|
||||||
mcs = __xen_mc_entry(sizeof(*op));
|
|
||||||
|
|
||||||
op = mcs.args;
|
|
||||||
op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
|
|
||||||
op->arg1.mfn = mfn;
|
|
||||||
|
|
||||||
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
|
||||||
|
|
||||||
if (kernel) {
|
|
||||||
percpu_write(xen_cr3, cr3);
|
|
||||||
|
|
||||||
/* Update xen_current_cr3 once the batch has actually
|
|
||||||
been submitted. */
|
|
||||||
xen_mc_callback(set_current_cr3, (void *)cr3);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_write_cr3(unsigned long cr3)
|
|
||||||
{
|
|
||||||
BUG_ON(preemptible());
|
|
||||||
|
|
||||||
xen_mc_batch(); /* disables interrupts */
|
|
||||||
|
|
||||||
/* Update while interrupts are disabled, so its atomic with
|
|
||||||
respect to ipis */
|
|
||||||
percpu_write(xen_cr3, cr3);
|
|
||||||
|
|
||||||
__xen_write_cr3(true, cr3);
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
{
|
|
||||||
pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
|
|
||||||
if (user_pgd)
|
|
||||||
__xen_write_cr3(false, __pa(user_pgd));
|
|
||||||
else
|
|
||||||
__xen_write_cr3(false, 0);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
|
|
||||||
}
|
|
||||||
|
|
||||||
static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
|
static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
|
||||||
{
|
{
|
||||||
int ret;
|
int ret;
|
||||||
|
@ -822,185 +638,6 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Early in boot, while setting up the initial pagetable, assume
|
|
||||||
everything is pinned. */
|
|
||||||
static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
|
|
||||||
{
|
|
||||||
#ifdef CONFIG_FLATMEM
|
|
||||||
BUG_ON(mem_map); /* should only be used early */
|
|
||||||
#endif
|
|
||||||
make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Early release_pte assumes that all pts are pinned, since there's
|
|
||||||
only init_mm and anything attached to that is pinned. */
|
|
||||||
static void xen_release_pte_init(unsigned long pfn)
|
|
||||||
{
|
|
||||||
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
|
|
||||||
}
|
|
||||||
|
|
||||||
static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
|
|
||||||
{
|
|
||||||
struct mmuext_op op;
|
|
||||||
op.cmd = cmd;
|
|
||||||
op.arg1.mfn = pfn_to_mfn(pfn);
|
|
||||||
if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
|
|
||||||
BUG();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* This needs to make sure the new pte page is pinned iff its being
|
|
||||||
attached to a pinned pagetable. */
|
|
||||||
static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level)
|
|
||||||
{
|
|
||||||
struct page *page = pfn_to_page(pfn);
|
|
||||||
|
|
||||||
if (PagePinned(virt_to_page(mm->pgd))) {
|
|
||||||
SetPagePinned(page);
|
|
||||||
|
|
||||||
vm_unmap_aliases();
|
|
||||||
if (!PageHighMem(page)) {
|
|
||||||
make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn)));
|
|
||||||
if (level == PT_PTE && USE_SPLIT_PTLOCKS)
|
|
||||||
pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
|
|
||||||
} else {
|
|
||||||
/* make sure there are no stray mappings of
|
|
||||||
this page */
|
|
||||||
kmap_flush_unused();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_alloc_ptpage(mm, pfn, PT_PTE);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_alloc_ptpage(mm, pfn, PT_PMD);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int xen_pgd_alloc(struct mm_struct *mm)
|
|
||||||
{
|
|
||||||
pgd_t *pgd = mm->pgd;
|
|
||||||
int ret = 0;
|
|
||||||
|
|
||||||
BUG_ON(PagePinned(virt_to_page(pgd)));
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
{
|
|
||||||
struct page *page = virt_to_page(pgd);
|
|
||||||
pgd_t *user_pgd;
|
|
||||||
|
|
||||||
BUG_ON(page->private != 0);
|
|
||||||
|
|
||||||
ret = -ENOMEM;
|
|
||||||
|
|
||||||
user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
|
|
||||||
page->private = (unsigned long)user_pgd;
|
|
||||||
|
|
||||||
if (user_pgd != NULL) {
|
|
||||||
user_pgd[pgd_index(VSYSCALL_START)] =
|
|
||||||
__pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
|
|
||||||
ret = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
|
||||||
{
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
pgd_t *user_pgd = xen_get_user_pgd(pgd);
|
|
||||||
|
|
||||||
if (user_pgd)
|
|
||||||
free_page((unsigned long)user_pgd);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/* This should never happen until we're OK to use struct page */
|
|
||||||
static void xen_release_ptpage(unsigned long pfn, unsigned level)
|
|
||||||
{
|
|
||||||
struct page *page = pfn_to_page(pfn);
|
|
||||||
|
|
||||||
if (PagePinned(page)) {
|
|
||||||
if (!PageHighMem(page)) {
|
|
||||||
if (level == PT_PTE && USE_SPLIT_PTLOCKS)
|
|
||||||
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
|
|
||||||
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
|
|
||||||
}
|
|
||||||
ClearPagePinned(page);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_release_pte(unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_release_ptpage(pfn, PT_PTE);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_release_pmd(unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_release_ptpage(pfn, PT_PMD);
|
|
||||||
}
|
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
|
||||||
static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_alloc_ptpage(mm, pfn, PT_PUD);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void xen_release_pud(unsigned long pfn)
|
|
||||||
{
|
|
||||||
xen_release_ptpage(pfn, PT_PUD);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef CONFIG_HIGHPTE
|
|
||||||
static void *xen_kmap_atomic_pte(struct page *page, enum km_type type)
|
|
||||||
{
|
|
||||||
pgprot_t prot = PAGE_KERNEL;
|
|
||||||
|
|
||||||
if (PagePinned(page))
|
|
||||||
prot = PAGE_KERNEL_RO;
|
|
||||||
|
|
||||||
if (0 && PageHighMem(page))
|
|
||||||
printk("mapping highpte %lx type %d prot %s\n",
|
|
||||||
page_to_pfn(page), type,
|
|
||||||
(unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ");
|
|
||||||
|
|
||||||
return kmap_atomic_prot(page, type, prot);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_32
|
|
||||||
static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
|
|
||||||
{
|
|
||||||
/* If there's an existing pte, then don't allow _PAGE_RW to be set */
|
|
||||||
if (pte_val_ma(*ptep) & _PAGE_PRESENT)
|
|
||||||
pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
|
|
||||||
pte_val_ma(pte));
|
|
||||||
|
|
||||||
return pte;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Init-time set_pte while constructing initial pagetables, which
|
|
||||||
doesn't allow RO pagetable pages to be remapped RW */
|
|
||||||
static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
|
|
||||||
{
|
|
||||||
pte = mask_rw_pte(ptep, pte);
|
|
||||||
|
|
||||||
xen_set_pte(ptep, pte);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static __init void xen_pagetable_setup_start(pgd_t *base)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void xen_setup_shared_info(void)
|
void xen_setup_shared_info(void)
|
||||||
{
|
{
|
||||||
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
|
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
|
||||||
|
@ -1021,37 +658,6 @@ void xen_setup_shared_info(void)
|
||||||
xen_setup_mfn_list_list();
|
xen_setup_mfn_list_list();
|
||||||
}
|
}
|
||||||
|
|
||||||
static __init void xen_pagetable_setup_done(pgd_t *base)
|
|
||||||
{
|
|
||||||
xen_setup_shared_info();
|
|
||||||
}
|
|
||||||
|
|
||||||
static __init void xen_post_allocator_init(void)
|
|
||||||
{
|
|
||||||
pv_mmu_ops.set_pte = xen_set_pte;
|
|
||||||
pv_mmu_ops.set_pmd = xen_set_pmd;
|
|
||||||
pv_mmu_ops.set_pud = xen_set_pud;
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
|
||||||
pv_mmu_ops.set_pgd = xen_set_pgd;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* This will work as long as patching hasn't happened yet
|
|
||||||
(which it hasn't) */
|
|
||||||
pv_mmu_ops.alloc_pte = xen_alloc_pte;
|
|
||||||
pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
|
|
||||||
pv_mmu_ops.release_pte = xen_release_pte;
|
|
||||||
pv_mmu_ops.release_pmd = xen_release_pmd;
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
|
||||||
pv_mmu_ops.alloc_pud = xen_alloc_pud;
|
|
||||||
pv_mmu_ops.release_pud = xen_release_pud;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
SetPagePinned(virt_to_page(level3_user_vsyscall));
|
|
||||||
#endif
|
|
||||||
xen_mark_init_mm_pinned();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* This is called once we have the cpu_possible_map */
|
/* This is called once we have the cpu_possible_map */
|
||||||
void xen_setup_vcpu_info_placement(void)
|
void xen_setup_vcpu_info_placement(void)
|
||||||
{
|
{
|
||||||
|
@ -1065,10 +671,10 @@ void xen_setup_vcpu_info_placement(void)
|
||||||
if (have_vcpu_info_placement) {
|
if (have_vcpu_info_placement) {
|
||||||
printk(KERN_INFO "Xen: using vcpu_info placement\n");
|
printk(KERN_INFO "Xen: using vcpu_info placement\n");
|
||||||
|
|
||||||
pv_irq_ops.save_fl = xen_save_fl_direct;
|
pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
|
||||||
pv_irq_ops.restore_fl = xen_restore_fl_direct;
|
pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
|
||||||
pv_irq_ops.irq_disable = xen_irq_disable_direct;
|
pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
|
||||||
pv_irq_ops.irq_enable = xen_irq_enable_direct;
|
pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
|
||||||
pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
|
pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1126,49 +732,6 @@ static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void xen_set_fixmap(unsigned idx, unsigned long phys, pgprot_t prot)
|
|
||||||
{
|
|
||||||
pte_t pte;
|
|
||||||
|
|
||||||
phys >>= PAGE_SHIFT;
|
|
||||||
|
|
||||||
switch (idx) {
|
|
||||||
case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
|
|
||||||
#ifdef CONFIG_X86_F00F_BUG
|
|
||||||
case FIX_F00F_IDT:
|
|
||||||
#endif
|
|
||||||
#ifdef CONFIG_X86_32
|
|
||||||
case FIX_WP_TEST:
|
|
||||||
case FIX_VDSO:
|
|
||||||
# ifdef CONFIG_HIGHMEM
|
|
||||||
case FIX_KMAP_BEGIN ... FIX_KMAP_END:
|
|
||||||
# endif
|
|
||||||
#else
|
|
||||||
case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
|
|
||||||
#endif
|
|
||||||
#ifdef CONFIG_X86_LOCAL_APIC
|
|
||||||
case FIX_APIC_BASE: /* maps dummy local APIC */
|
|
||||||
#endif
|
|
||||||
pte = pfn_pte(phys, prot);
|
|
||||||
break;
|
|
||||||
|
|
||||||
default:
|
|
||||||
pte = mfn_pte(phys, prot);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
__native_set_fixmap(idx, pte);
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
/* Replicate changes to map the vsyscall page into the user
|
|
||||||
pagetable vsyscall mapping. */
|
|
||||||
if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) {
|
|
||||||
unsigned long vaddr = __fix_to_virt(idx);
|
|
||||||
set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
static const struct pv_info xen_info __initdata = {
|
static const struct pv_info xen_info __initdata = {
|
||||||
.paravirt_enabled = 1,
|
.paravirt_enabled = 1,
|
||||||
.shared_kernel_pmd = 0,
|
.shared_kernel_pmd = 0,
|
||||||
|
@ -1264,86 +827,6 @@ static const struct pv_apic_ops xen_apic_ops __initdata = {
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
static const struct pv_mmu_ops xen_mmu_ops __initdata = {
|
|
||||||
.pagetable_setup_start = xen_pagetable_setup_start,
|
|
||||||
.pagetable_setup_done = xen_pagetable_setup_done,
|
|
||||||
|
|
||||||
.read_cr2 = xen_read_cr2,
|
|
||||||
.write_cr2 = xen_write_cr2,
|
|
||||||
|
|
||||||
.read_cr3 = xen_read_cr3,
|
|
||||||
.write_cr3 = xen_write_cr3,
|
|
||||||
|
|
||||||
.flush_tlb_user = xen_flush_tlb,
|
|
||||||
.flush_tlb_kernel = xen_flush_tlb,
|
|
||||||
.flush_tlb_single = xen_flush_tlb_single,
|
|
||||||
.flush_tlb_others = xen_flush_tlb_others,
|
|
||||||
|
|
||||||
.pte_update = paravirt_nop,
|
|
||||||
.pte_update_defer = paravirt_nop,
|
|
||||||
|
|
||||||
.pgd_alloc = xen_pgd_alloc,
|
|
||||||
.pgd_free = xen_pgd_free,
|
|
||||||
|
|
||||||
.alloc_pte = xen_alloc_pte_init,
|
|
||||||
.release_pte = xen_release_pte_init,
|
|
||||||
.alloc_pmd = xen_alloc_pte_init,
|
|
||||||
.alloc_pmd_clone = paravirt_nop,
|
|
||||||
.release_pmd = xen_release_pte_init,
|
|
||||||
|
|
||||||
#ifdef CONFIG_HIGHPTE
|
|
||||||
.kmap_atomic_pte = xen_kmap_atomic_pte,
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
.set_pte = xen_set_pte,
|
|
||||||
#else
|
|
||||||
.set_pte = xen_set_pte_init,
|
|
||||||
#endif
|
|
||||||
.set_pte_at = xen_set_pte_at,
|
|
||||||
.set_pmd = xen_set_pmd_hyper,
|
|
||||||
|
|
||||||
.ptep_modify_prot_start = __ptep_modify_prot_start,
|
|
||||||
.ptep_modify_prot_commit = __ptep_modify_prot_commit,
|
|
||||||
|
|
||||||
.pte_val = xen_pte_val,
|
|
||||||
.pgd_val = xen_pgd_val,
|
|
||||||
|
|
||||||
.make_pte = xen_make_pte,
|
|
||||||
.make_pgd = xen_make_pgd,
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_PAE
|
|
||||||
.set_pte_atomic = xen_set_pte_atomic,
|
|
||||||
.set_pte_present = xen_set_pte_at,
|
|
||||||
.pte_clear = xen_pte_clear,
|
|
||||||
.pmd_clear = xen_pmd_clear,
|
|
||||||
#endif /* CONFIG_X86_PAE */
|
|
||||||
.set_pud = xen_set_pud_hyper,
|
|
||||||
|
|
||||||
.make_pmd = xen_make_pmd,
|
|
||||||
.pmd_val = xen_pmd_val,
|
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
|
||||||
.pud_val = xen_pud_val,
|
|
||||||
.make_pud = xen_make_pud,
|
|
||||||
.set_pgd = xen_set_pgd_hyper,
|
|
||||||
|
|
||||||
.alloc_pud = xen_alloc_pte_init,
|
|
||||||
.release_pud = xen_release_pte_init,
|
|
||||||
#endif /* PAGETABLE_LEVELS == 4 */
|
|
||||||
|
|
||||||
.activate_mm = xen_activate_mm,
|
|
||||||
.dup_mmap = xen_dup_mmap,
|
|
||||||
.exit_mmap = xen_exit_mmap,
|
|
||||||
|
|
||||||
.lazy_mode = {
|
|
||||||
.enter = paravirt_enter_lazy_mmu,
|
|
||||||
.leave = xen_leave_lazy,
|
|
||||||
},
|
|
||||||
|
|
||||||
.set_fixmap = xen_set_fixmap,
|
|
||||||
};
|
|
||||||
|
|
||||||
static void xen_reboot(int reason)
|
static void xen_reboot(int reason)
|
||||||
{
|
{
|
||||||
struct sched_shutdown r = { .reason = reason };
|
struct sched_shutdown r = { .reason = reason };
|
||||||
|
@ -1386,223 +869,6 @@ static const struct machine_ops __initdata xen_machine_ops = {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
static void __init xen_reserve_top(void)
|
|
||||||
{
|
|
||||||
#ifdef CONFIG_X86_32
|
|
||||||
unsigned long top = HYPERVISOR_VIRT_START;
|
|
||||||
struct xen_platform_parameters pp;
|
|
||||||
|
|
||||||
if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
|
|
||||||
top = pp.virt_start;
|
|
||||||
|
|
||||||
reserve_top_address(-top);
|
|
||||||
#endif /* CONFIG_X86_32 */
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Like __va(), but returns address in the kernel mapping (which is
|
|
||||||
* all we have until the physical memory mapping has been set up.
|
|
||||||
*/
|
|
||||||
static void *__ka(phys_addr_t paddr)
|
|
||||||
{
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
return (void *)(paddr + __START_KERNEL_map);
|
|
||||||
#else
|
|
||||||
return __va(paddr);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Convert a machine address to physical address */
|
|
||||||
static unsigned long m2p(phys_addr_t maddr)
|
|
||||||
{
|
|
||||||
phys_addr_t paddr;
|
|
||||||
|
|
||||||
maddr &= PTE_PFN_MASK;
|
|
||||||
paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
|
|
||||||
|
|
||||||
return paddr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Convert a machine address to kernel virtual */
|
|
||||||
static void *m2v(phys_addr_t maddr)
|
|
||||||
{
|
|
||||||
return __ka(m2p(maddr));
|
|
||||||
}
|
|
||||||
|
|
||||||
static void set_page_prot(void *addr, pgprot_t prot)
|
|
||||||
{
|
|
||||||
unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
|
|
||||||
pte_t pte = pfn_pte(pfn, prot);
|
|
||||||
|
|
||||||
if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
|
|
||||||
BUG();
|
|
||||||
}
|
|
||||||
|
|
||||||
static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
|
|
||||||
{
|
|
||||||
unsigned pmdidx, pteidx;
|
|
||||||
unsigned ident_pte;
|
|
||||||
unsigned long pfn;
|
|
||||||
|
|
||||||
ident_pte = 0;
|
|
||||||
pfn = 0;
|
|
||||||
for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
|
|
||||||
pte_t *pte_page;
|
|
||||||
|
|
||||||
/* Reuse or allocate a page of ptes */
|
|
||||||
if (pmd_present(pmd[pmdidx]))
|
|
||||||
pte_page = m2v(pmd[pmdidx].pmd);
|
|
||||||
else {
|
|
||||||
/* Check for free pte pages */
|
|
||||||
if (ident_pte == ARRAY_SIZE(level1_ident_pgt))
|
|
||||||
break;
|
|
||||||
|
|
||||||
pte_page = &level1_ident_pgt[ident_pte];
|
|
||||||
ident_pte += PTRS_PER_PTE;
|
|
||||||
|
|
||||||
pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Install mappings */
|
|
||||||
for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
|
|
||||||
pte_t pte;
|
|
||||||
|
|
||||||
if (pfn > max_pfn_mapped)
|
|
||||||
max_pfn_mapped = pfn;
|
|
||||||
|
|
||||||
if (!pte_none(pte_page[pteidx]))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
|
|
||||||
pte_page[pteidx] = pte;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
|
|
||||||
set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
|
|
||||||
|
|
||||||
set_page_prot(pmd, PAGE_KERNEL_RO);
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
static void convert_pfn_mfn(void *v)
|
|
||||||
{
|
|
||||||
pte_t *pte = v;
|
|
||||||
int i;
|
|
||||||
|
|
||||||
/* All levels are converted the same way, so just treat them
|
|
||||||
as ptes. */
|
|
||||||
for (i = 0; i < PTRS_PER_PTE; i++)
|
|
||||||
pte[i] = xen_make_pte(pte[i].pte);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Set up the inital kernel pagetable.
|
|
||||||
*
|
|
||||||
* We can construct this by grafting the Xen provided pagetable into
|
|
||||||
* head_64.S's preconstructed pagetables. We copy the Xen L2's into
|
|
||||||
* level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
|
|
||||||
* means that only the kernel has a physical mapping to start with -
|
|
||||||
* but that's enough to get __va working. We need to fill in the rest
|
|
||||||
* of the physical mapping once some sort of allocator has been set
|
|
||||||
* up.
|
|
||||||
*/
|
|
||||||
static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
|
|
||||||
unsigned long max_pfn)
|
|
||||||
{
|
|
||||||
pud_t *l3;
|
|
||||||
pmd_t *l2;
|
|
||||||
|
|
||||||
/* Zap identity mapping */
|
|
||||||
init_level4_pgt[0] = __pgd(0);
|
|
||||||
|
|
||||||
/* Pre-constructed entries are in pfn, so convert to mfn */
|
|
||||||
convert_pfn_mfn(init_level4_pgt);
|
|
||||||
convert_pfn_mfn(level3_ident_pgt);
|
|
||||||
convert_pfn_mfn(level3_kernel_pgt);
|
|
||||||
|
|
||||||
l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
|
|
||||||
l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
|
|
||||||
|
|
||||||
memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
|
||||||
memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
|
||||||
|
|
||||||
l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
|
|
||||||
l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
|
|
||||||
memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
|
||||||
|
|
||||||
/* Set up identity map */
|
|
||||||
xen_map_identity_early(level2_ident_pgt, max_pfn);
|
|
||||||
|
|
||||||
/* Make pagetable pieces RO */
|
|
||||||
set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
|
|
||||||
|
|
||||||
/* Pin down new L4 */
|
|
||||||
pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
|
|
||||||
PFN_DOWN(__pa_symbol(init_level4_pgt)));
|
|
||||||
|
|
||||||
/* Unpin Xen-provided one */
|
|
||||||
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
|
|
||||||
|
|
||||||
/* Switch over */
|
|
||||||
pgd = init_level4_pgt;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* At this stage there can be no user pgd, and no page
|
|
||||||
* structure to attach it to, so make sure we just set kernel
|
|
||||||
* pgd.
|
|
||||||
*/
|
|
||||||
xen_mc_batch();
|
|
||||||
__xen_write_cr3(true, __pa(pgd));
|
|
||||||
xen_mc_issue(PARAVIRT_LAZY_CPU);
|
|
||||||
|
|
||||||
reserve_early(__pa(xen_start_info->pt_base),
|
|
||||||
__pa(xen_start_info->pt_base +
|
|
||||||
xen_start_info->nr_pt_frames * PAGE_SIZE),
|
|
||||||
"XEN PAGETABLES");
|
|
||||||
|
|
||||||
return pgd;
|
|
||||||
}
|
|
||||||
#else /* !CONFIG_X86_64 */
|
|
||||||
static pmd_t level2_kernel_pgt[PTRS_PER_PMD] __page_aligned_bss;
|
|
||||||
|
|
||||||
static __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
|
|
||||||
unsigned long max_pfn)
|
|
||||||
{
|
|
||||||
pmd_t *kernel_pmd;
|
|
||||||
|
|
||||||
init_pg_tables_start = __pa(pgd);
|
|
||||||
init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;
|
|
||||||
max_pfn_mapped = PFN_DOWN(init_pg_tables_end + 512*1024);
|
|
||||||
|
|
||||||
kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
|
|
||||||
memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
|
|
||||||
|
|
||||||
xen_map_identity_early(level2_kernel_pgt, max_pfn);
|
|
||||||
|
|
||||||
memcpy(swapper_pg_dir, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
|
|
||||||
set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY],
|
|
||||||
__pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT));
|
|
||||||
|
|
||||||
set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
|
|
||||||
set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
|
|
||||||
|
|
||||||
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
|
|
||||||
|
|
||||||
xen_write_cr3(__pa(swapper_pg_dir));
|
|
||||||
|
|
||||||
pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir)));
|
|
||||||
|
|
||||||
return swapper_pg_dir;
|
|
||||||
}
|
|
||||||
#endif /* CONFIG_X86_64 */
|
|
||||||
|
|
||||||
/* First C function to be called on Xen boot */
|
/* First C function to be called on Xen boot */
|
||||||
asmlinkage void __init xen_start_kernel(void)
|
asmlinkage void __init xen_start_kernel(void)
|
||||||
{
|
{
|
||||||
|
@ -1642,9 +908,18 @@ asmlinkage void __init xen_start_kernel(void)
|
||||||
machine_ops = xen_machine_ops;
|
machine_ops = xen_machine_ops;
|
||||||
|
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_64
|
||||||
/* Disable until direct per-cpu data access. */
|
/*
|
||||||
have_vcpu_info_placement = 0;
|
* Setup percpu state. We only need to do this for 64-bit
|
||||||
|
* because 32-bit already has %fs set properly.
|
||||||
|
*/
|
||||||
|
load_percpu_segment(0);
|
||||||
#endif
|
#endif
|
||||||
|
/*
|
||||||
|
* The only reliable way to retain the initial address of the
|
||||||
|
* percpu gdt_page is to remember it here, so we can go and
|
||||||
|
* mark it RW later, when the initial percpu area is freed.
|
||||||
|
*/
|
||||||
|
xen_initial_gdt = &per_cpu(gdt_page, 0);
|
||||||
|
|
||||||
xen_smp_init();
|
xen_smp_init();
|
||||||
|
|
||||||
|
|
|
@ -50,6 +50,7 @@ static unsigned long xen_save_fl(void)
|
||||||
*/
|
*/
|
||||||
return (-flags) & X86_EFLAGS_IF;
|
return (-flags) & X86_EFLAGS_IF;
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl);
|
||||||
|
|
||||||
static void xen_restore_fl(unsigned long flags)
|
static void xen_restore_fl(unsigned long flags)
|
||||||
{
|
{
|
||||||
|
@ -76,6 +77,7 @@ static void xen_restore_fl(unsigned long flags)
|
||||||
xen_force_evtchn_callback();
|
xen_force_evtchn_callback();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_restore_fl);
|
||||||
|
|
||||||
static void xen_irq_disable(void)
|
static void xen_irq_disable(void)
|
||||||
{
|
{
|
||||||
|
@ -86,6 +88,7 @@ static void xen_irq_disable(void)
|
||||||
percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
|
percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
|
||||||
preempt_enable_no_resched();
|
preempt_enable_no_resched();
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
|
||||||
|
|
||||||
static void xen_irq_enable(void)
|
static void xen_irq_enable(void)
|
||||||
{
|
{
|
||||||
|
@ -106,6 +109,7 @@ static void xen_irq_enable(void)
|
||||||
if (unlikely(vcpu->evtchn_upcall_pending))
|
if (unlikely(vcpu->evtchn_upcall_pending))
|
||||||
xen_force_evtchn_callback();
|
xen_force_evtchn_callback();
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_enable);
|
||||||
|
|
||||||
static void xen_safe_halt(void)
|
static void xen_safe_halt(void)
|
||||||
{
|
{
|
||||||
|
@ -124,10 +128,12 @@ static void xen_halt(void)
|
||||||
|
|
||||||
static const struct pv_irq_ops xen_irq_ops __initdata = {
|
static const struct pv_irq_ops xen_irq_ops __initdata = {
|
||||||
.init_IRQ = __xen_init_IRQ,
|
.init_IRQ = __xen_init_IRQ,
|
||||||
.save_fl = xen_save_fl,
|
|
||||||
.restore_fl = xen_restore_fl,
|
.save_fl = PV_CALLEE_SAVE(xen_save_fl),
|
||||||
.irq_disable = xen_irq_disable,
|
.restore_fl = PV_CALLEE_SAVE(xen_restore_fl),
|
||||||
.irq_enable = xen_irq_enable,
|
.irq_disable = PV_CALLEE_SAVE(xen_irq_disable),
|
||||||
|
.irq_enable = PV_CALLEE_SAVE(xen_irq_enable),
|
||||||
|
|
||||||
.safe_halt = xen_safe_halt,
|
.safe_halt = xen_safe_halt,
|
||||||
.halt = xen_halt,
|
.halt = xen_halt,
|
||||||
#ifdef CONFIG_X86_64
|
#ifdef CONFIG_X86_64
|
||||||
|
|
|
@ -47,6 +47,7 @@
|
||||||
#include <asm/tlbflush.h>
|
#include <asm/tlbflush.h>
|
||||||
#include <asm/fixmap.h>
|
#include <asm/fixmap.h>
|
||||||
#include <asm/mmu_context.h>
|
#include <asm/mmu_context.h>
|
||||||
|
#include <asm/setup.h>
|
||||||
#include <asm/paravirt.h>
|
#include <asm/paravirt.h>
|
||||||
#include <asm/linkage.h>
|
#include <asm/linkage.h>
|
||||||
|
|
||||||
|
@ -55,6 +56,8 @@
|
||||||
|
|
||||||
#include <xen/page.h>
|
#include <xen/page.h>
|
||||||
#include <xen/interface/xen.h>
|
#include <xen/interface/xen.h>
|
||||||
|
#include <xen/interface/version.h>
|
||||||
|
#include <xen/hvc-console.h>
|
||||||
|
|
||||||
#include "multicalls.h"
|
#include "multicalls.h"
|
||||||
#include "mmu.h"
|
#include "mmu.h"
|
||||||
|
@ -114,6 +117,37 @@ static inline void check_zero(void)
|
||||||
|
|
||||||
#endif /* CONFIG_XEN_DEBUG_FS */
|
#endif /* CONFIG_XEN_DEBUG_FS */
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Identity map, in addition to plain kernel map. This needs to be
|
||||||
|
* large enough to allocate page table pages to allocate the rest.
|
||||||
|
* Each page can map 2MB.
|
||||||
|
*/
|
||||||
|
static pte_t level1_ident_pgt[PTRS_PER_PTE * 4] __page_aligned_bss;
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
/* l3 pud for userspace vsyscall mapping */
|
||||||
|
static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
|
||||||
|
#endif /* CONFIG_X86_64 */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Note about cr3 (pagetable base) values:
|
||||||
|
*
|
||||||
|
* xen_cr3 contains the current logical cr3 value; it contains the
|
||||||
|
* last set cr3. This may not be the current effective cr3, because
|
||||||
|
* its update may be being lazily deferred. However, a vcpu looking
|
||||||
|
* at its own cr3 can use this value knowing that it everything will
|
||||||
|
* be self-consistent.
|
||||||
|
*
|
||||||
|
* xen_current_cr3 contains the actual vcpu cr3; it is set once the
|
||||||
|
* hypercall to set the vcpu cr3 is complete (so it may be a little
|
||||||
|
* out of date, but it will never be set early). If one vcpu is
|
||||||
|
* looking at another vcpu's cr3 value, it should use this variable.
|
||||||
|
*/
|
||||||
|
DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
|
||||||
|
DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Just beyond the highest usermode address. STACK_TOP_MAX has a
|
* Just beyond the highest usermode address. STACK_TOP_MAX has a
|
||||||
* redzone above it, so round it up to a PGD boundary.
|
* redzone above it, so round it up to a PGD boundary.
|
||||||
|
@ -458,28 +492,33 @@ pteval_t xen_pte_val(pte_t pte)
|
||||||
{
|
{
|
||||||
return pte_mfn_to_pfn(pte.pte);
|
return pte_mfn_to_pfn(pte.pte);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
|
||||||
|
|
||||||
pgdval_t xen_pgd_val(pgd_t pgd)
|
pgdval_t xen_pgd_val(pgd_t pgd)
|
||||||
{
|
{
|
||||||
return pte_mfn_to_pfn(pgd.pgd);
|
return pte_mfn_to_pfn(pgd.pgd);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
|
||||||
|
|
||||||
pte_t xen_make_pte(pteval_t pte)
|
pte_t xen_make_pte(pteval_t pte)
|
||||||
{
|
{
|
||||||
pte = pte_pfn_to_mfn(pte);
|
pte = pte_pfn_to_mfn(pte);
|
||||||
return native_make_pte(pte);
|
return native_make_pte(pte);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
|
||||||
|
|
||||||
pgd_t xen_make_pgd(pgdval_t pgd)
|
pgd_t xen_make_pgd(pgdval_t pgd)
|
||||||
{
|
{
|
||||||
pgd = pte_pfn_to_mfn(pgd);
|
pgd = pte_pfn_to_mfn(pgd);
|
||||||
return native_make_pgd(pgd);
|
return native_make_pgd(pgd);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
|
||||||
|
|
||||||
pmdval_t xen_pmd_val(pmd_t pmd)
|
pmdval_t xen_pmd_val(pmd_t pmd)
|
||||||
{
|
{
|
||||||
return pte_mfn_to_pfn(pmd.pmd);
|
return pte_mfn_to_pfn(pmd.pmd);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
|
||||||
|
|
||||||
void xen_set_pud_hyper(pud_t *ptr, pud_t val)
|
void xen_set_pud_hyper(pud_t *ptr, pud_t val)
|
||||||
{
|
{
|
||||||
|
@ -556,12 +595,14 @@ pmd_t xen_make_pmd(pmdval_t pmd)
|
||||||
pmd = pte_pfn_to_mfn(pmd);
|
pmd = pte_pfn_to_mfn(pmd);
|
||||||
return native_make_pmd(pmd);
|
return native_make_pmd(pmd);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
|
||||||
|
|
||||||
#if PAGETABLE_LEVELS == 4
|
#if PAGETABLE_LEVELS == 4
|
||||||
pudval_t xen_pud_val(pud_t pud)
|
pudval_t xen_pud_val(pud_t pud)
|
||||||
{
|
{
|
||||||
return pte_mfn_to_pfn(pud.pud);
|
return pte_mfn_to_pfn(pud.pud);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
|
||||||
|
|
||||||
pud_t xen_make_pud(pudval_t pud)
|
pud_t xen_make_pud(pudval_t pud)
|
||||||
{
|
{
|
||||||
|
@ -569,6 +610,7 @@ pud_t xen_make_pud(pudval_t pud)
|
||||||
|
|
||||||
return native_make_pud(pud);
|
return native_make_pud(pud);
|
||||||
}
|
}
|
||||||
|
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
|
||||||
|
|
||||||
pgd_t *xen_get_user_pgd(pgd_t *pgd)
|
pgd_t *xen_get_user_pgd(pgd_t *pgd)
|
||||||
{
|
{
|
||||||
|
@ -1152,6 +1194,709 @@ void xen_exit_mmap(struct mm_struct *mm)
|
||||||
spin_unlock(&mm->page_table_lock);
|
spin_unlock(&mm->page_table_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static __init void xen_pagetable_setup_start(pgd_t *base)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
static __init void xen_pagetable_setup_done(pgd_t *base)
|
||||||
|
{
|
||||||
|
xen_setup_shared_info();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_write_cr2(unsigned long cr2)
|
||||||
|
{
|
||||||
|
percpu_read(xen_vcpu)->arch.cr2 = cr2;
|
||||||
|
}
|
||||||
|
|
||||||
|
static unsigned long xen_read_cr2(void)
|
||||||
|
{
|
||||||
|
return percpu_read(xen_vcpu)->arch.cr2;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long xen_read_cr2_direct(void)
|
||||||
|
{
|
||||||
|
return percpu_read(xen_vcpu_info.arch.cr2);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_flush_tlb(void)
|
||||||
|
{
|
||||||
|
struct mmuext_op *op;
|
||||||
|
struct multicall_space mcs;
|
||||||
|
|
||||||
|
preempt_disable();
|
||||||
|
|
||||||
|
mcs = xen_mc_entry(sizeof(*op));
|
||||||
|
|
||||||
|
op = mcs.args;
|
||||||
|
op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
|
||||||
|
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
||||||
|
|
||||||
|
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
||||||
|
|
||||||
|
preempt_enable();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_flush_tlb_single(unsigned long addr)
|
||||||
|
{
|
||||||
|
struct mmuext_op *op;
|
||||||
|
struct multicall_space mcs;
|
||||||
|
|
||||||
|
preempt_disable();
|
||||||
|
|
||||||
|
mcs = xen_mc_entry(sizeof(*op));
|
||||||
|
op = mcs.args;
|
||||||
|
op->cmd = MMUEXT_INVLPG_LOCAL;
|
||||||
|
op->arg1.linear_addr = addr & PAGE_MASK;
|
||||||
|
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
||||||
|
|
||||||
|
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
||||||
|
|
||||||
|
preempt_enable();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_flush_tlb_others(const struct cpumask *cpus,
|
||||||
|
struct mm_struct *mm, unsigned long va)
|
||||||
|
{
|
||||||
|
struct {
|
||||||
|
struct mmuext_op op;
|
||||||
|
DECLARE_BITMAP(mask, NR_CPUS);
|
||||||
|
} *args;
|
||||||
|
struct multicall_space mcs;
|
||||||
|
|
||||||
|
BUG_ON(cpumask_empty(cpus));
|
||||||
|
BUG_ON(!mm);
|
||||||
|
|
||||||
|
mcs = xen_mc_entry(sizeof(*args));
|
||||||
|
args = mcs.args;
|
||||||
|
args->op.arg2.vcpumask = to_cpumask(args->mask);
|
||||||
|
|
||||||
|
/* Remove us, and any offline CPUS. */
|
||||||
|
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
|
||||||
|
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
|
||||||
|
if (unlikely(cpumask_empty(to_cpumask(args->mask))))
|
||||||
|
goto issue;
|
||||||
|
|
||||||
|
if (va == TLB_FLUSH_ALL) {
|
||||||
|
args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
|
||||||
|
} else {
|
||||||
|
args->op.cmd = MMUEXT_INVLPG_MULTI;
|
||||||
|
args->op.arg1.linear_addr = va;
|
||||||
|
}
|
||||||
|
|
||||||
|
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
|
||||||
|
|
||||||
|
issue:
|
||||||
|
xen_mc_issue(PARAVIRT_LAZY_MMU);
|
||||||
|
}
|
||||||
|
|
||||||
|
static unsigned long xen_read_cr3(void)
|
||||||
|
{
|
||||||
|
return percpu_read(xen_cr3);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void set_current_cr3(void *v)
|
||||||
|
{
|
||||||
|
percpu_write(xen_current_cr3, (unsigned long)v);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void __xen_write_cr3(bool kernel, unsigned long cr3)
|
||||||
|
{
|
||||||
|
struct mmuext_op *op;
|
||||||
|
struct multicall_space mcs;
|
||||||
|
unsigned long mfn;
|
||||||
|
|
||||||
|
if (cr3)
|
||||||
|
mfn = pfn_to_mfn(PFN_DOWN(cr3));
|
||||||
|
else
|
||||||
|
mfn = 0;
|
||||||
|
|
||||||
|
WARN_ON(mfn == 0 && kernel);
|
||||||
|
|
||||||
|
mcs = __xen_mc_entry(sizeof(*op));
|
||||||
|
|
||||||
|
op = mcs.args;
|
||||||
|
op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
|
||||||
|
op->arg1.mfn = mfn;
|
||||||
|
|
||||||
|
MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
|
||||||
|
|
||||||
|
if (kernel) {
|
||||||
|
percpu_write(xen_cr3, cr3);
|
||||||
|
|
||||||
|
/* Update xen_current_cr3 once the batch has actually
|
||||||
|
been submitted. */
|
||||||
|
xen_mc_callback(set_current_cr3, (void *)cr3);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_write_cr3(unsigned long cr3)
|
||||||
|
{
|
||||||
|
BUG_ON(preemptible());
|
||||||
|
|
||||||
|
xen_mc_batch(); /* disables interrupts */
|
||||||
|
|
||||||
|
/* Update while interrupts are disabled, so its atomic with
|
||||||
|
respect to ipis */
|
||||||
|
percpu_write(xen_cr3, cr3);
|
||||||
|
|
||||||
|
__xen_write_cr3(true, cr3);
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
{
|
||||||
|
pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
|
||||||
|
if (user_pgd)
|
||||||
|
__xen_write_cr3(false, __pa(user_pgd));
|
||||||
|
else
|
||||||
|
__xen_write_cr3(false, 0);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
|
||||||
|
}
|
||||||
|
|
||||||
|
static int xen_pgd_alloc(struct mm_struct *mm)
|
||||||
|
{
|
||||||
|
pgd_t *pgd = mm->pgd;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
BUG_ON(PagePinned(virt_to_page(pgd)));
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
{
|
||||||
|
struct page *page = virt_to_page(pgd);
|
||||||
|
pgd_t *user_pgd;
|
||||||
|
|
||||||
|
BUG_ON(page->private != 0);
|
||||||
|
|
||||||
|
ret = -ENOMEM;
|
||||||
|
|
||||||
|
user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
|
||||||
|
page->private = (unsigned long)user_pgd;
|
||||||
|
|
||||||
|
if (user_pgd != NULL) {
|
||||||
|
user_pgd[pgd_index(VSYSCALL_START)] =
|
||||||
|
__pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
|
||||||
|
ret = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
pgd_t *user_pgd = xen_get_user_pgd(pgd);
|
||||||
|
|
||||||
|
if (user_pgd)
|
||||||
|
free_page((unsigned long)user_pgd);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_HIGHPTE
|
||||||
|
static void *xen_kmap_atomic_pte(struct page *page, enum km_type type)
|
||||||
|
{
|
||||||
|
pgprot_t prot = PAGE_KERNEL;
|
||||||
|
|
||||||
|
if (PagePinned(page))
|
||||||
|
prot = PAGE_KERNEL_RO;
|
||||||
|
|
||||||
|
if (0 && PageHighMem(page))
|
||||||
|
printk("mapping highpte %lx type %d prot %s\n",
|
||||||
|
page_to_pfn(page), type,
|
||||||
|
(unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ");
|
||||||
|
|
||||||
|
return kmap_atomic_prot(page, type, prot);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
|
||||||
|
{
|
||||||
|
/* If there's an existing pte, then don't allow _PAGE_RW to be set */
|
||||||
|
if (pte_val_ma(*ptep) & _PAGE_PRESENT)
|
||||||
|
pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
|
||||||
|
pte_val_ma(pte));
|
||||||
|
|
||||||
|
return pte;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Init-time set_pte while constructing initial pagetables, which
|
||||||
|
doesn't allow RO pagetable pages to be remapped RW */
|
||||||
|
static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
|
||||||
|
{
|
||||||
|
pte = mask_rw_pte(ptep, pte);
|
||||||
|
|
||||||
|
xen_set_pte(ptep, pte);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Early in boot, while setting up the initial pagetable, assume
|
||||||
|
everything is pinned. */
|
||||||
|
static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_FLATMEM
|
||||||
|
BUG_ON(mem_map); /* should only be used early */
|
||||||
|
#endif
|
||||||
|
make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Early release_pte assumes that all pts are pinned, since there's
|
||||||
|
only init_mm and anything attached to that is pinned. */
|
||||||
|
static void xen_release_pte_init(unsigned long pfn)
|
||||||
|
{
|
||||||
|
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
|
||||||
|
{
|
||||||
|
struct mmuext_op op;
|
||||||
|
op.cmd = cmd;
|
||||||
|
op.arg1.mfn = pfn_to_mfn(pfn);
|
||||||
|
if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
|
||||||
|
BUG();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* This needs to make sure the new pte page is pinned iff its being
|
||||||
|
attached to a pinned pagetable. */
|
||||||
|
static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level)
|
||||||
|
{
|
||||||
|
struct page *page = pfn_to_page(pfn);
|
||||||
|
|
||||||
|
if (PagePinned(virt_to_page(mm->pgd))) {
|
||||||
|
SetPagePinned(page);
|
||||||
|
|
||||||
|
vm_unmap_aliases();
|
||||||
|
if (!PageHighMem(page)) {
|
||||||
|
make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn)));
|
||||||
|
if (level == PT_PTE && USE_SPLIT_PTLOCKS)
|
||||||
|
pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
|
||||||
|
} else {
|
||||||
|
/* make sure there are no stray mappings of
|
||||||
|
this page */
|
||||||
|
kmap_flush_unused();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_alloc_ptpage(mm, pfn, PT_PTE);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_alloc_ptpage(mm, pfn, PT_PMD);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* This should never happen until we're OK to use struct page */
|
||||||
|
static void xen_release_ptpage(unsigned long pfn, unsigned level)
|
||||||
|
{
|
||||||
|
struct page *page = pfn_to_page(pfn);
|
||||||
|
|
||||||
|
if (PagePinned(page)) {
|
||||||
|
if (!PageHighMem(page)) {
|
||||||
|
if (level == PT_PTE && USE_SPLIT_PTLOCKS)
|
||||||
|
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
|
||||||
|
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
|
||||||
|
}
|
||||||
|
ClearPagePinned(page);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_release_pte(unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_release_ptpage(pfn, PT_PTE);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_release_pmd(unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_release_ptpage(pfn, PT_PMD);
|
||||||
|
}
|
||||||
|
|
||||||
|
#if PAGETABLE_LEVELS == 4
|
||||||
|
static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_alloc_ptpage(mm, pfn, PT_PUD);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void xen_release_pud(unsigned long pfn)
|
||||||
|
{
|
||||||
|
xen_release_ptpage(pfn, PT_PUD);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
void __init xen_reserve_top(void)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
unsigned long top = HYPERVISOR_VIRT_START;
|
||||||
|
struct xen_platform_parameters pp;
|
||||||
|
|
||||||
|
if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
|
||||||
|
top = pp.virt_start;
|
||||||
|
|
||||||
|
reserve_top_address(-top);
|
||||||
|
#endif /* CONFIG_X86_32 */
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Like __va(), but returns address in the kernel mapping (which is
|
||||||
|
* all we have until the physical memory mapping has been set up.
|
||||||
|
*/
|
||||||
|
static void *__ka(phys_addr_t paddr)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
return (void *)(paddr + __START_KERNEL_map);
|
||||||
|
#else
|
||||||
|
return __va(paddr);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Convert a machine address to physical address */
|
||||||
|
static unsigned long m2p(phys_addr_t maddr)
|
||||||
|
{
|
||||||
|
phys_addr_t paddr;
|
||||||
|
|
||||||
|
maddr &= PTE_PFN_MASK;
|
||||||
|
paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
|
||||||
|
|
||||||
|
return paddr;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Convert a machine address to kernel virtual */
|
||||||
|
static void *m2v(phys_addr_t maddr)
|
||||||
|
{
|
||||||
|
return __ka(m2p(maddr));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void set_page_prot(void *addr, pgprot_t prot)
|
||||||
|
{
|
||||||
|
unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
|
||||||
|
pte_t pte = pfn_pte(pfn, prot);
|
||||||
|
|
||||||
|
if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
|
||||||
|
BUG();
|
||||||
|
}
|
||||||
|
|
||||||
|
static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
|
||||||
|
{
|
||||||
|
unsigned pmdidx, pteidx;
|
||||||
|
unsigned ident_pte;
|
||||||
|
unsigned long pfn;
|
||||||
|
|
||||||
|
ident_pte = 0;
|
||||||
|
pfn = 0;
|
||||||
|
for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
|
||||||
|
pte_t *pte_page;
|
||||||
|
|
||||||
|
/* Reuse or allocate a page of ptes */
|
||||||
|
if (pmd_present(pmd[pmdidx]))
|
||||||
|
pte_page = m2v(pmd[pmdidx].pmd);
|
||||||
|
else {
|
||||||
|
/* Check for free pte pages */
|
||||||
|
if (ident_pte == ARRAY_SIZE(level1_ident_pgt))
|
||||||
|
break;
|
||||||
|
|
||||||
|
pte_page = &level1_ident_pgt[ident_pte];
|
||||||
|
ident_pte += PTRS_PER_PTE;
|
||||||
|
|
||||||
|
pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Install mappings */
|
||||||
|
for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
|
||||||
|
pte_t pte;
|
||||||
|
|
||||||
|
if (pfn > max_pfn_mapped)
|
||||||
|
max_pfn_mapped = pfn;
|
||||||
|
|
||||||
|
if (!pte_none(pte_page[pteidx]))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
|
||||||
|
pte_page[pteidx] = pte;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
|
||||||
|
set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
|
||||||
|
|
||||||
|
set_page_prot(pmd, PAGE_KERNEL_RO);
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
static void convert_pfn_mfn(void *v)
|
||||||
|
{
|
||||||
|
pte_t *pte = v;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
/* All levels are converted the same way, so just treat them
|
||||||
|
as ptes. */
|
||||||
|
for (i = 0; i < PTRS_PER_PTE; i++)
|
||||||
|
pte[i] = xen_make_pte(pte[i].pte);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Set up the inital kernel pagetable.
|
||||||
|
*
|
||||||
|
* We can construct this by grafting the Xen provided pagetable into
|
||||||
|
* head_64.S's preconstructed pagetables. We copy the Xen L2's into
|
||||||
|
* level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
|
||||||
|
* means that only the kernel has a physical mapping to start with -
|
||||||
|
* but that's enough to get __va working. We need to fill in the rest
|
||||||
|
* of the physical mapping once some sort of allocator has been set
|
||||||
|
* up.
|
||||||
|
*/
|
||||||
|
__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
|
||||||
|
unsigned long max_pfn)
|
||||||
|
{
|
||||||
|
pud_t *l3;
|
||||||
|
pmd_t *l2;
|
||||||
|
|
||||||
|
/* Zap identity mapping */
|
||||||
|
init_level4_pgt[0] = __pgd(0);
|
||||||
|
|
||||||
|
/* Pre-constructed entries are in pfn, so convert to mfn */
|
||||||
|
convert_pfn_mfn(init_level4_pgt);
|
||||||
|
convert_pfn_mfn(level3_ident_pgt);
|
||||||
|
convert_pfn_mfn(level3_kernel_pgt);
|
||||||
|
|
||||||
|
l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
|
||||||
|
l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
|
||||||
|
|
||||||
|
memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
||||||
|
memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
||||||
|
|
||||||
|
l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
|
||||||
|
l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
|
||||||
|
memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
|
||||||
|
|
||||||
|
/* Set up identity map */
|
||||||
|
xen_map_identity_early(level2_ident_pgt, max_pfn);
|
||||||
|
|
||||||
|
/* Make pagetable pieces RO */
|
||||||
|
set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
|
||||||
|
|
||||||
|
/* Pin down new L4 */
|
||||||
|
pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
|
||||||
|
PFN_DOWN(__pa_symbol(init_level4_pgt)));
|
||||||
|
|
||||||
|
/* Unpin Xen-provided one */
|
||||||
|
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
|
||||||
|
|
||||||
|
/* Switch over */
|
||||||
|
pgd = init_level4_pgt;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* At this stage there can be no user pgd, and no page
|
||||||
|
* structure to attach it to, so make sure we just set kernel
|
||||||
|
* pgd.
|
||||||
|
*/
|
||||||
|
xen_mc_batch();
|
||||||
|
__xen_write_cr3(true, __pa(pgd));
|
||||||
|
xen_mc_issue(PARAVIRT_LAZY_CPU);
|
||||||
|
|
||||||
|
reserve_early(__pa(xen_start_info->pt_base),
|
||||||
|
__pa(xen_start_info->pt_base +
|
||||||
|
xen_start_info->nr_pt_frames * PAGE_SIZE),
|
||||||
|
"XEN PAGETABLES");
|
||||||
|
|
||||||
|
return pgd;
|
||||||
|
}
|
||||||
|
#else /* !CONFIG_X86_64 */
|
||||||
|
static pmd_t level2_kernel_pgt[PTRS_PER_PMD] __page_aligned_bss;
|
||||||
|
|
||||||
|
__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
|
||||||
|
unsigned long max_pfn)
|
||||||
|
{
|
||||||
|
pmd_t *kernel_pmd;
|
||||||
|
|
||||||
|
init_pg_tables_start = __pa(pgd);
|
||||||
|
init_pg_tables_end = __pa(pgd) + xen_start_info->nr_pt_frames*PAGE_SIZE;
|
||||||
|
max_pfn_mapped = PFN_DOWN(init_pg_tables_end + 512*1024);
|
||||||
|
|
||||||
|
kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
|
||||||
|
memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
|
||||||
|
|
||||||
|
xen_map_identity_early(level2_kernel_pgt, max_pfn);
|
||||||
|
|
||||||
|
memcpy(swapper_pg_dir, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
|
||||||
|
set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY],
|
||||||
|
__pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT));
|
||||||
|
|
||||||
|
set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
|
||||||
|
set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
|
||||||
|
|
||||||
|
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
|
||||||
|
|
||||||
|
xen_write_cr3(__pa(swapper_pg_dir));
|
||||||
|
|
||||||
|
pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir)));
|
||||||
|
|
||||||
|
return swapper_pg_dir;
|
||||||
|
}
|
||||||
|
#endif /* CONFIG_X86_64 */
|
||||||
|
|
||||||
|
static void xen_set_fixmap(unsigned idx, unsigned long phys, pgprot_t prot)
|
||||||
|
{
|
||||||
|
pte_t pte;
|
||||||
|
|
||||||
|
phys >>= PAGE_SHIFT;
|
||||||
|
|
||||||
|
switch (idx) {
|
||||||
|
case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
|
||||||
|
#ifdef CONFIG_X86_F00F_BUG
|
||||||
|
case FIX_F00F_IDT:
|
||||||
|
#endif
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
case FIX_WP_TEST:
|
||||||
|
case FIX_VDSO:
|
||||||
|
# ifdef CONFIG_HIGHMEM
|
||||||
|
case FIX_KMAP_BEGIN ... FIX_KMAP_END:
|
||||||
|
# endif
|
||||||
|
#else
|
||||||
|
case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
|
||||||
|
#endif
|
||||||
|
#ifdef CONFIG_X86_LOCAL_APIC
|
||||||
|
case FIX_APIC_BASE: /* maps dummy local APIC */
|
||||||
|
#endif
|
||||||
|
pte = pfn_pte(phys, prot);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
pte = mfn_pte(phys, prot);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
__native_set_fixmap(idx, pte);
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
/* Replicate changes to map the vsyscall page into the user
|
||||||
|
pagetable vsyscall mapping. */
|
||||||
|
if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) {
|
||||||
|
unsigned long vaddr = __fix_to_virt(idx);
|
||||||
|
set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
__init void xen_post_allocator_init(void)
|
||||||
|
{
|
||||||
|
pv_mmu_ops.set_pte = xen_set_pte;
|
||||||
|
pv_mmu_ops.set_pmd = xen_set_pmd;
|
||||||
|
pv_mmu_ops.set_pud = xen_set_pud;
|
||||||
|
#if PAGETABLE_LEVELS == 4
|
||||||
|
pv_mmu_ops.set_pgd = xen_set_pgd;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* This will work as long as patching hasn't happened yet
|
||||||
|
(which it hasn't) */
|
||||||
|
pv_mmu_ops.alloc_pte = xen_alloc_pte;
|
||||||
|
pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
|
||||||
|
pv_mmu_ops.release_pte = xen_release_pte;
|
||||||
|
pv_mmu_ops.release_pmd = xen_release_pmd;
|
||||||
|
#if PAGETABLE_LEVELS == 4
|
||||||
|
pv_mmu_ops.alloc_pud = xen_alloc_pud;
|
||||||
|
pv_mmu_ops.release_pud = xen_release_pud;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
SetPagePinned(virt_to_page(level3_user_vsyscall));
|
||||||
|
#endif
|
||||||
|
xen_mark_init_mm_pinned();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
const struct pv_mmu_ops xen_mmu_ops __initdata = {
|
||||||
|
.pagetable_setup_start = xen_pagetable_setup_start,
|
||||||
|
.pagetable_setup_done = xen_pagetable_setup_done,
|
||||||
|
|
||||||
|
.read_cr2 = xen_read_cr2,
|
||||||
|
.write_cr2 = xen_write_cr2,
|
||||||
|
|
||||||
|
.read_cr3 = xen_read_cr3,
|
||||||
|
.write_cr3 = xen_write_cr3,
|
||||||
|
|
||||||
|
.flush_tlb_user = xen_flush_tlb,
|
||||||
|
.flush_tlb_kernel = xen_flush_tlb,
|
||||||
|
.flush_tlb_single = xen_flush_tlb_single,
|
||||||
|
.flush_tlb_others = xen_flush_tlb_others,
|
||||||
|
|
||||||
|
.pte_update = paravirt_nop,
|
||||||
|
.pte_update_defer = paravirt_nop,
|
||||||
|
|
||||||
|
.pgd_alloc = xen_pgd_alloc,
|
||||||
|
.pgd_free = xen_pgd_free,
|
||||||
|
|
||||||
|
.alloc_pte = xen_alloc_pte_init,
|
||||||
|
.release_pte = xen_release_pte_init,
|
||||||
|
.alloc_pmd = xen_alloc_pte_init,
|
||||||
|
.alloc_pmd_clone = paravirt_nop,
|
||||||
|
.release_pmd = xen_release_pte_init,
|
||||||
|
|
||||||
|
#ifdef CONFIG_HIGHPTE
|
||||||
|
.kmap_atomic_pte = xen_kmap_atomic_pte,
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
.set_pte = xen_set_pte,
|
||||||
|
#else
|
||||||
|
.set_pte = xen_set_pte_init,
|
||||||
|
#endif
|
||||||
|
.set_pte_at = xen_set_pte_at,
|
||||||
|
.set_pmd = xen_set_pmd_hyper,
|
||||||
|
|
||||||
|
.ptep_modify_prot_start = __ptep_modify_prot_start,
|
||||||
|
.ptep_modify_prot_commit = __ptep_modify_prot_commit,
|
||||||
|
|
||||||
|
.pte_val = PV_CALLEE_SAVE(xen_pte_val),
|
||||||
|
.pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
|
||||||
|
|
||||||
|
.make_pte = PV_CALLEE_SAVE(xen_make_pte),
|
||||||
|
.make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
|
||||||
|
|
||||||
|
#ifdef CONFIG_X86_PAE
|
||||||
|
.set_pte_atomic = xen_set_pte_atomic,
|
||||||
|
.set_pte_present = xen_set_pte_at,
|
||||||
|
.pte_clear = xen_pte_clear,
|
||||||
|
.pmd_clear = xen_pmd_clear,
|
||||||
|
#endif /* CONFIG_X86_PAE */
|
||||||
|
.set_pud = xen_set_pud_hyper,
|
||||||
|
|
||||||
|
.make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
|
||||||
|
.pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
|
||||||
|
|
||||||
|
#if PAGETABLE_LEVELS == 4
|
||||||
|
.pud_val = PV_CALLEE_SAVE(xen_pud_val),
|
||||||
|
.make_pud = PV_CALLEE_SAVE(xen_make_pud),
|
||||||
|
.set_pgd = xen_set_pgd_hyper,
|
||||||
|
|
||||||
|
.alloc_pud = xen_alloc_pte_init,
|
||||||
|
.release_pud = xen_release_pte_init,
|
||||||
|
#endif /* PAGETABLE_LEVELS == 4 */
|
||||||
|
|
||||||
|
.activate_mm = xen_activate_mm,
|
||||||
|
.dup_mmap = xen_dup_mmap,
|
||||||
|
.exit_mmap = xen_exit_mmap,
|
||||||
|
|
||||||
|
.lazy_mode = {
|
||||||
|
.enter = paravirt_enter_lazy_mmu,
|
||||||
|
.leave = xen_leave_lazy,
|
||||||
|
},
|
||||||
|
|
||||||
|
.set_fixmap = xen_set_fixmap,
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
#ifdef CONFIG_XEN_DEBUG_FS
|
#ifdef CONFIG_XEN_DEBUG_FS
|
||||||
|
|
||||||
static struct dentry *d_mmu_debug;
|
static struct dentry *d_mmu_debug;
|
||||||
|
|
|
@ -54,4 +54,7 @@ pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t
|
||||||
void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
|
void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
|
||||||
pte_t *ptep, pte_t pte);
|
pte_t *ptep, pte_t pte);
|
||||||
|
|
||||||
|
unsigned long xen_read_cr2_direct(void);
|
||||||
|
|
||||||
|
extern const struct pv_mmu_ops xen_mmu_ops;
|
||||||
#endif /* _XEN_MMU_H */
|
#endif /* _XEN_MMU_H */
|
||||||
|
|
|
@ -170,7 +170,7 @@ static void __init xen_smp_prepare_boot_cpu(void)
|
||||||
|
|
||||||
/* We've switched to the "real" per-cpu gdt, so make sure the
|
/* We've switched to the "real" per-cpu gdt, so make sure the
|
||||||
old memory can be recycled */
|
old memory can be recycled */
|
||||||
make_lowmem_page_readwrite(&per_cpu_var(gdt_page));
|
make_lowmem_page_readwrite(xen_initial_gdt);
|
||||||
|
|
||||||
xen_setup_vcpu_info_placement();
|
xen_setup_vcpu_info_placement();
|
||||||
}
|
}
|
||||||
|
@ -235,6 +235,8 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
|
||||||
ctxt->user_regs.ss = __KERNEL_DS;
|
ctxt->user_regs.ss = __KERNEL_DS;
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
ctxt->user_regs.fs = __KERNEL_PERCPU;
|
ctxt->user_regs.fs = __KERNEL_PERCPU;
|
||||||
|
#else
|
||||||
|
ctxt->gs_base_kernel = per_cpu_offset(cpu);
|
||||||
#endif
|
#endif
|
||||||
ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
|
ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
|
||||||
ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
|
ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
|
||||||
|
@ -284,6 +286,9 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
|
||||||
irq_ctx_init(cpu);
|
irq_ctx_init(cpu);
|
||||||
#else
|
#else
|
||||||
clear_tsk_thread_flag(idle, TIF_FORK);
|
clear_tsk_thread_flag(idle, TIF_FORK);
|
||||||
|
per_cpu(kernel_stack, cpu) =
|
||||||
|
(unsigned long)task_stack_page(idle) -
|
||||||
|
KERNEL_STACK_OFFSET + THREAD_SIZE;
|
||||||
#endif
|
#endif
|
||||||
xen_setup_timer(cpu);
|
xen_setup_timer(cpu);
|
||||||
xen_init_lock_cpu(cpu);
|
xen_init_lock_cpu(cpu);
|
||||||
|
|
140
arch/x86/xen/xen-asm.S
Normal file
140
arch/x86/xen/xen-asm.S
Normal file
|
@ -0,0 +1,140 @@
|
||||||
|
/*
|
||||||
|
Asm versions of Xen pv-ops, suitable for either direct use or inlining.
|
||||||
|
The inline versions are the same as the direct-use versions, with the
|
||||||
|
pre- and post-amble chopped off.
|
||||||
|
|
||||||
|
This code is encoded for size rather than absolute efficiency,
|
||||||
|
with a view to being able to inline as much as possible.
|
||||||
|
|
||||||
|
We only bother with direct forms (ie, vcpu in percpu data) of
|
||||||
|
the operations here; the indirect forms are better handled in
|
||||||
|
C, since they're generally too large to inline anyway.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <asm/asm-offsets.h>
|
||||||
|
#include <asm/percpu.h>
|
||||||
|
#include <asm/processor-flags.h>
|
||||||
|
|
||||||
|
#include "xen-asm.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
Enable events. This clears the event mask and tests the pending
|
||||||
|
event status with one and operation. If there are pending
|
||||||
|
events, then enter the hypervisor to get them handled.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_irq_enable_direct)
|
||||||
|
/* Unmask events */
|
||||||
|
movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
|
||||||
|
/* Preempt here doesn't matter because that will deal with
|
||||||
|
any pending interrupts. The pending check may end up being
|
||||||
|
run on the wrong CPU, but that doesn't hurt. */
|
||||||
|
|
||||||
|
/* Test for pending */
|
||||||
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
||||||
|
jz 1f
|
||||||
|
|
||||||
|
2: call check_events
|
||||||
|
1:
|
||||||
|
ENDPATCH(xen_irq_enable_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_irq_enable_direct)
|
||||||
|
RELOC(xen_irq_enable_direct, 2b+1)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
Disabling events is simply a matter of making the event mask
|
||||||
|
non-zero.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_irq_disable_direct)
|
||||||
|
movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
ENDPATCH(xen_irq_disable_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_irq_disable_direct)
|
||||||
|
RELOC(xen_irq_disable_direct, 0)
|
||||||
|
|
||||||
|
/*
|
||||||
|
(xen_)save_fl is used to get the current interrupt enable status.
|
||||||
|
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
||||||
|
may be set in the return value. We take advantage of this by
|
||||||
|
making sure that X86_EFLAGS_IF has the right value (and other bits
|
||||||
|
in that byte are 0), but other bits in the return value are
|
||||||
|
undefined. We need to toggle the state of the bit, because
|
||||||
|
Xen and x86 use opposite senses (mask vs enable).
|
||||||
|
*/
|
||||||
|
ENTRY(xen_save_fl_direct)
|
||||||
|
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
setz %ah
|
||||||
|
addb %ah,%ah
|
||||||
|
ENDPATCH(xen_save_fl_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_save_fl_direct)
|
||||||
|
RELOC(xen_save_fl_direct, 0)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
In principle the caller should be passing us a value return
|
||||||
|
from xen_save_fl_direct, but for robustness sake we test only
|
||||||
|
the X86_EFLAGS_IF flag rather than the whole byte. After
|
||||||
|
setting the interrupt mask state, it checks for unmasked
|
||||||
|
pending events and enters the hypervisor to get them delivered
|
||||||
|
if so.
|
||||||
|
*/
|
||||||
|
ENTRY(xen_restore_fl_direct)
|
||||||
|
#ifdef CONFIG_X86_64
|
||||||
|
testw $X86_EFLAGS_IF, %di
|
||||||
|
#else
|
||||||
|
testb $X86_EFLAGS_IF>>8, %ah
|
||||||
|
#endif
|
||||||
|
setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
||||||
|
/* Preempt here doesn't matter because that will deal with
|
||||||
|
any pending interrupts. The pending check may end up being
|
||||||
|
run on the wrong CPU, but that doesn't hurt. */
|
||||||
|
|
||||||
|
/* check for unmasked and pending */
|
||||||
|
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
||||||
|
jz 1f
|
||||||
|
2: call check_events
|
||||||
|
1:
|
||||||
|
ENDPATCH(xen_restore_fl_direct)
|
||||||
|
ret
|
||||||
|
ENDPROC(xen_restore_fl_direct)
|
||||||
|
RELOC(xen_restore_fl_direct, 2b+1)
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
Force an event check by making a hypercall,
|
||||||
|
but preserve regs before making the call.
|
||||||
|
*/
|
||||||
|
check_events:
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
push %eax
|
||||||
|
push %ecx
|
||||||
|
push %edx
|
||||||
|
call xen_force_evtchn_callback
|
||||||
|
pop %edx
|
||||||
|
pop %ecx
|
||||||
|
pop %eax
|
||||||
|
#else
|
||||||
|
push %rax
|
||||||
|
push %rcx
|
||||||
|
push %rdx
|
||||||
|
push %rsi
|
||||||
|
push %rdi
|
||||||
|
push %r8
|
||||||
|
push %r9
|
||||||
|
push %r10
|
||||||
|
push %r11
|
||||||
|
call xen_force_evtchn_callback
|
||||||
|
pop %r11
|
||||||
|
pop %r10
|
||||||
|
pop %r9
|
||||||
|
pop %r8
|
||||||
|
pop %rdi
|
||||||
|
pop %rsi
|
||||||
|
pop %rdx
|
||||||
|
pop %rcx
|
||||||
|
pop %rax
|
||||||
|
#endif
|
||||||
|
ret
|
||||||
|
|
12
arch/x86/xen/xen-asm.h
Normal file
12
arch/x86/xen/xen-asm.h
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
#ifndef _XEN_XEN_ASM_H
|
||||||
|
#define _XEN_XEN_ASM_H
|
||||||
|
|
||||||
|
#include <linux/linkage.h>
|
||||||
|
|
||||||
|
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
||||||
|
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
||||||
|
|
||||||
|
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
||||||
|
#define XEN_EFLAGS_NMI 0x80000000
|
||||||
|
|
||||||
|
#endif
|
|
@ -11,101 +11,28 @@
|
||||||
generally too large to inline anyway.
|
generally too large to inline anyway.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
//#include <asm/asm-offsets.h>
|
||||||
|
|
||||||
#include <asm/asm-offsets.h>
|
|
||||||
#include <asm/thread_info.h>
|
#include <asm/thread_info.h>
|
||||||
#include <asm/percpu.h>
|
|
||||||
#include <asm/processor-flags.h>
|
#include <asm/processor-flags.h>
|
||||||
#include <asm/segment.h>
|
#include <asm/segment.h>
|
||||||
|
|
||||||
#include <xen/interface/xen.h>
|
#include <xen/interface/xen.h>
|
||||||
|
|
||||||
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
#include "xen-asm.h"
|
||||||
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
|
||||||
|
|
||||||
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
|
||||||
#define XEN_EFLAGS_NMI 0x80000000
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Enable events. This clears the event mask and tests the pending
|
Force an event check by making a hypercall,
|
||||||
event status with one and operation. If there are pending
|
but preserve regs before making the call.
|
||||||
events, then enter the hypervisor to get them handled.
|
|
||||||
*/
|
*/
|
||||||
ENTRY(xen_irq_enable_direct)
|
check_events:
|
||||||
/* Unmask events */
|
push %eax
|
||||||
movb $0, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
push %ecx
|
||||||
|
push %edx
|
||||||
/* Preempt here doesn't matter because that will deal with
|
call xen_force_evtchn_callback
|
||||||
any pending interrupts. The pending check may end up being
|
pop %edx
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
pop %ecx
|
||||||
|
pop %eax
|
||||||
/* Test for pending */
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_irq_enable_direct)
|
|
||||||
ret
|
ret
|
||||||
ENDPROC(xen_irq_enable_direct)
|
|
||||||
RELOC(xen_irq_enable_direct, 2b+1)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Disabling events is simply a matter of making the event mask
|
|
||||||
non-zero.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_disable_direct)
|
|
||||||
movb $1, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
ENDPATCH(xen_irq_disable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_disable_direct)
|
|
||||||
RELOC(xen_irq_disable_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
(xen_)save_fl is used to get the current interrupt enable status.
|
|
||||||
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
||||||
may be set in the return value. We take advantage of this by
|
|
||||||
making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
||||||
in that byte are 0), but other bits in the return value are
|
|
||||||
undefined. We need to toggle the state of the bit, because
|
|
||||||
Xen and x86 use opposite senses (mask vs enable).
|
|
||||||
*/
|
|
||||||
ENTRY(xen_save_fl_direct)
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
setz %ah
|
|
||||||
addb %ah,%ah
|
|
||||||
ENDPATCH(xen_save_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_save_fl_direct)
|
|
||||||
RELOC(xen_save_fl_direct, 0)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
In principle the caller should be passing us a value return
|
|
||||||
from xen_save_fl_direct, but for robustness sake we test only
|
|
||||||
the X86_EFLAGS_IF flag rather than the whole byte. After
|
|
||||||
setting the interrupt mask state, it checks for unmasked
|
|
||||||
pending events and enters the hypervisor to get them delivered
|
|
||||||
if so.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_restore_fl_direct)
|
|
||||||
testb $X86_EFLAGS_IF>>8, %ah
|
|
||||||
setz PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* check for unmasked and pending */
|
|
||||||
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_restore_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_restore_fl_direct)
|
|
||||||
RELOC(xen_restore_fl_direct, 2b+1)
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
We can't use sysexit directly, because we're not running in ring0.
|
We can't use sysexit directly, because we're not running in ring0.
|
||||||
|
@ -289,17 +216,3 @@ ENTRY(xen_iret_crit_fixup)
|
||||||
lea 4(%edi),%esp /* point esp to new frame */
|
lea 4(%edi),%esp /* point esp to new frame */
|
||||||
2: jmp xen_do_upcall
|
2: jmp xen_do_upcall
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Force an event check by making a hypercall,
|
|
||||||
but preserve regs before making the call.
|
|
||||||
*/
|
|
||||||
check_events:
|
|
||||||
push %eax
|
|
||||||
push %ecx
|
|
||||||
push %edx
|
|
||||||
call xen_force_evtchn_callback
|
|
||||||
pop %edx
|
|
||||||
pop %ecx
|
|
||||||
pop %eax
|
|
||||||
ret
|
|
||||||
|
|
|
@ -11,142 +11,14 @@
|
||||||
generally too large to inline anyway.
|
generally too large to inline anyway.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <linux/linkage.h>
|
|
||||||
|
|
||||||
#include <asm/asm-offsets.h>
|
|
||||||
#include <asm/processor-flags.h>
|
|
||||||
#include <asm/errno.h>
|
#include <asm/errno.h>
|
||||||
#include <asm/segment.h>
|
|
||||||
#include <asm/percpu.h>
|
#include <asm/percpu.h>
|
||||||
|
#include <asm/processor-flags.h>
|
||||||
|
#include <asm/segment.h>
|
||||||
|
|
||||||
#include <xen/interface/xen.h>
|
#include <xen/interface/xen.h>
|
||||||
|
|
||||||
#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
|
#include "xen-asm.h"
|
||||||
#define ENDPATCH(x) .globl x##_end; x##_end=.
|
|
||||||
|
|
||||||
/* Pseudo-flag used for virtual NMI, which we don't implement yet */
|
|
||||||
#define XEN_EFLAGS_NMI 0x80000000
|
|
||||||
|
|
||||||
#if 1
|
|
||||||
/*
|
|
||||||
FIXME: x86_64 now can support direct access to percpu variables
|
|
||||||
via a segment override. Update xen accordingly.
|
|
||||||
*/
|
|
||||||
#define BUG ud2a
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*
|
|
||||||
Enable events. This clears the event mask and tests the pending
|
|
||||||
event status with one and operation. If there are pending
|
|
||||||
events, then enter the hypervisor to get them handled.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_enable_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
/* Unmask events */
|
|
||||||
movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* Test for pending */
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_irq_enable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_enable_direct)
|
|
||||||
RELOC(xen_irq_enable_direct, 2b+1)
|
|
||||||
|
|
||||||
/*
|
|
||||||
Disabling events is simply a matter of making the event mask
|
|
||||||
non-zero.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_irq_disable_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
ENDPATCH(xen_irq_disable_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_irq_disable_direct)
|
|
||||||
RELOC(xen_irq_disable_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
(xen_)save_fl is used to get the current interrupt enable status.
|
|
||||||
Callers expect the status to be in X86_EFLAGS_IF, and other bits
|
|
||||||
may be set in the return value. We take advantage of this by
|
|
||||||
making sure that X86_EFLAGS_IF has the right value (and other bits
|
|
||||||
in that byte are 0), but other bits in the return value are
|
|
||||||
undefined. We need to toggle the state of the bit, because
|
|
||||||
Xen and x86 use opposite senses (mask vs enable).
|
|
||||||
*/
|
|
||||||
ENTRY(xen_save_fl_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
setz %ah
|
|
||||||
addb %ah,%ah
|
|
||||||
ENDPATCH(xen_save_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_save_fl_direct)
|
|
||||||
RELOC(xen_save_fl_direct, 0)
|
|
||||||
|
|
||||||
/*
|
|
||||||
In principle the caller should be passing us a value return
|
|
||||||
from xen_save_fl_direct, but for robustness sake we test only
|
|
||||||
the X86_EFLAGS_IF flag rather than the whole byte. After
|
|
||||||
setting the interrupt mask state, it checks for unmasked
|
|
||||||
pending events and enters the hypervisor to get them delivered
|
|
||||||
if so.
|
|
||||||
*/
|
|
||||||
ENTRY(xen_restore_fl_direct)
|
|
||||||
BUG
|
|
||||||
|
|
||||||
testb $X86_EFLAGS_IF>>8, %ah
|
|
||||||
setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
|
|
||||||
/* Preempt here doesn't matter because that will deal with
|
|
||||||
any pending interrupts. The pending check may end up being
|
|
||||||
run on the wrong CPU, but that doesn't hurt. */
|
|
||||||
|
|
||||||
/* check for unmasked and pending */
|
|
||||||
cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
|
|
||||||
jz 1f
|
|
||||||
2: call check_events
|
|
||||||
1:
|
|
||||||
ENDPATCH(xen_restore_fl_direct)
|
|
||||||
ret
|
|
||||||
ENDPROC(xen_restore_fl_direct)
|
|
||||||
RELOC(xen_restore_fl_direct, 2b+1)
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
Force an event check by making a hypercall,
|
|
||||||
but preserve regs before making the call.
|
|
||||||
*/
|
|
||||||
check_events:
|
|
||||||
push %rax
|
|
||||||
push %rcx
|
|
||||||
push %rdx
|
|
||||||
push %rsi
|
|
||||||
push %rdi
|
|
||||||
push %r8
|
|
||||||
push %r9
|
|
||||||
push %r10
|
|
||||||
push %r11
|
|
||||||
call xen_force_evtchn_callback
|
|
||||||
pop %r11
|
|
||||||
pop %r10
|
|
||||||
pop %r9
|
|
||||||
pop %r8
|
|
||||||
pop %rdi
|
|
||||||
pop %rsi
|
|
||||||
pop %rdx
|
|
||||||
pop %rcx
|
|
||||||
pop %rax
|
|
||||||
ret
|
|
||||||
|
|
||||||
ENTRY(xen_adjust_exception_frame)
|
ENTRY(xen_adjust_exception_frame)
|
||||||
mov 8+0(%rsp),%rcx
|
mov 8+0(%rsp),%rcx
|
||||||
|
|
|
@ -10,9 +10,12 @@
|
||||||
extern const char xen_hypervisor_callback[];
|
extern const char xen_hypervisor_callback[];
|
||||||
extern const char xen_failsafe_callback[];
|
extern const char xen_failsafe_callback[];
|
||||||
|
|
||||||
|
extern void *xen_initial_gdt;
|
||||||
|
|
||||||
struct trap_info;
|
struct trap_info;
|
||||||
void xen_copy_trap_info(struct trap_info *traps);
|
void xen_copy_trap_info(struct trap_info *traps);
|
||||||
|
|
||||||
|
DECLARE_PER_CPU(struct vcpu_info, xen_vcpu_info);
|
||||||
DECLARE_PER_CPU(unsigned long, xen_cr3);
|
DECLARE_PER_CPU(unsigned long, xen_cr3);
|
||||||
DECLARE_PER_CPU(unsigned long, xen_current_cr3);
|
DECLARE_PER_CPU(unsigned long, xen_current_cr3);
|
||||||
|
|
||||||
|
@ -22,6 +25,13 @@ extern struct shared_info *HYPERVISOR_shared_info;
|
||||||
|
|
||||||
void xen_setup_mfn_list_list(void);
|
void xen_setup_mfn_list_list(void);
|
||||||
void xen_setup_shared_info(void);
|
void xen_setup_shared_info(void);
|
||||||
|
void xen_setup_machphys_mapping(void);
|
||||||
|
pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
|
||||||
|
void xen_ident_map_ISA(void);
|
||||||
|
void xen_reserve_top(void);
|
||||||
|
|
||||||
|
void xen_leave_lazy(void);
|
||||||
|
void xen_post_allocator_init(void);
|
||||||
|
|
||||||
char * __init xen_memory_setup(void);
|
char * __init xen_memory_setup(void);
|
||||||
void __init xen_arch_setup(void);
|
void __init xen_arch_setup(void);
|
||||||
|
|
|
@ -445,24 +445,22 @@
|
||||||
* section in the linker script will go there too. @phdr should have
|
* section in the linker script will go there too. @phdr should have
|
||||||
* a leading colon.
|
* a leading colon.
|
||||||
*
|
*
|
||||||
* This macro defines three symbols, __per_cpu_load, __per_cpu_start
|
* Note that this macros defines __per_cpu_load as an absolute symbol.
|
||||||
* and __per_cpu_end. The first one is the vaddr of loaded percpu
|
* If there is no need to put the percpu section at a predetermined
|
||||||
* init data. __per_cpu_start equals @vaddr and __per_cpu_end is the
|
* address, use PERCPU().
|
||||||
* end offset.
|
|
||||||
*/
|
*/
|
||||||
#define PERCPU_VADDR(vaddr, phdr) \
|
#define PERCPU_VADDR(vaddr, phdr) \
|
||||||
VMLINUX_SYMBOL(__per_cpu_load_abs) = .; \
|
VMLINUX_SYMBOL(__per_cpu_load) = .; \
|
||||||
.data.percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load_abs) \
|
.data.percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load) \
|
||||||
- LOAD_OFFSET) { \
|
- LOAD_OFFSET) { \
|
||||||
VMLINUX_SYMBOL(__per_cpu_start) = .; \
|
VMLINUX_SYMBOL(__per_cpu_start) = .; \
|
||||||
VMLINUX_SYMBOL(__per_cpu_load) = LOADADDR(.data.percpu) + LOAD_OFFSET;\
|
|
||||||
*(.data.percpu.first) \
|
*(.data.percpu.first) \
|
||||||
*(.data.percpu.page_aligned) \
|
*(.data.percpu.page_aligned) \
|
||||||
*(.data.percpu) \
|
*(.data.percpu) \
|
||||||
*(.data.percpu.shared_aligned) \
|
*(.data.percpu.shared_aligned) \
|
||||||
VMLINUX_SYMBOL(__per_cpu_end) = .; \
|
VMLINUX_SYMBOL(__per_cpu_end) = .; \
|
||||||
} phdr \
|
} phdr \
|
||||||
. = VMLINUX_SYMBOL(__per_cpu_load_abs) + SIZEOF(.data.percpu);
|
. = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data.percpu);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PERCPU - define output section for percpu area, simple version
|
* PERCPU - define output section for percpu area, simple version
|
||||||
|
@ -471,7 +469,20 @@
|
||||||
* Align to @align and outputs output section for percpu area. This
|
* Align to @align and outputs output section for percpu area. This
|
||||||
* macro doesn't maniuplate @vaddr or @phdr and __per_cpu_load and
|
* macro doesn't maniuplate @vaddr or @phdr and __per_cpu_load and
|
||||||
* __per_cpu_start will be identical.
|
* __per_cpu_start will be identical.
|
||||||
|
*
|
||||||
|
* This macro is equivalent to ALIGN(align); PERCPU_VADDR( , ) except
|
||||||
|
* that __per_cpu_load is defined as a relative symbol against
|
||||||
|
* .data.percpu which is required for relocatable x86_32
|
||||||
|
* configuration.
|
||||||
*/
|
*/
|
||||||
#define PERCPU(align) \
|
#define PERCPU(align) \
|
||||||
. = ALIGN(align); \
|
. = ALIGN(align); \
|
||||||
PERCPU_VADDR( , )
|
.data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \
|
||||||
|
VMLINUX_SYMBOL(__per_cpu_load) = .; \
|
||||||
|
VMLINUX_SYMBOL(__per_cpu_start) = .; \
|
||||||
|
*(.data.percpu.first) \
|
||||||
|
*(.data.percpu.page_aligned) \
|
||||||
|
*(.data.percpu) \
|
||||||
|
*(.data.percpu.shared_aligned) \
|
||||||
|
VMLINUX_SYMBOL(__per_cpu_end) = .; \
|
||||||
|
}
|
||||||
|
|
Reference in a new issue