From d7c8f21a8cad0228c7c5ce2bb6dbd95d1ee49d13 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 30 Jan 2008 13:34:07 +0100 Subject: [PATCH] x86: cpa: move flush to cpa The set_memory_* and set_pages_* family of API's currently requires the callers to do a global tlb flush after the function call; forgetting this is a very nasty deathtrap. This patch moves the global tlb flush into each of the callers Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-gart_64.c | 1 - arch/x86/mm/init_32.c | 14 --- arch/x86/mm/init_64.c | 10 -- arch/x86/mm/ioremap.c | 2 - arch/x86/mm/pageattr.c | 137 ++++++++++++++-------------- drivers/char/agp/ali-agp.c | 2 - drivers/char/agp/i460-agp.c | 2 - drivers/char/agp/intel-agp.c | 5 - drivers/video/vermilion/vermilion.c | 6 -- include/asm-x86/agp.h | 6 +- include/asm-x86/cacheflush.h | 1 - sound/pci/intel8x0.c | 1 - 12 files changed, 72 insertions(+), 115 deletions(-) diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c index 8860c6eba8a..4d5cc718198 100644 --- a/arch/x86/kernel/pci-gart_64.c +++ b/arch/x86/kernel/pci-gart_64.c @@ -572,7 +572,6 @@ static __init int init_k8_gatt(struct agp_kern_info *info) panic("Cannot allocate GATT table"); if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT)) panic("Could not set GART PTEs to uncacheable pages"); - global_flush_tlb(); memset(gatt, 0, gatt_size); agp_gatt_table = gatt; diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index f7b941c3b2c..0d3369b900e 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -752,15 +752,11 @@ void mark_rodata_ro(void) printk("Write protecting the kernel text: %luk\n", size >> 10); #ifdef CONFIG_CPA_DEBUG - global_flush_tlb(); - printk("Testing CPA: Reverting %lx-%lx\n", start, start+size); set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT); - global_flush_tlb(); printk("Testing CPA: write protecting again\n"); set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); - global_flush_tlb(); #endif } #endif @@ -770,22 +766,12 @@ void mark_rodata_ro(void) printk("Write protecting the kernel read-only data: %luk\n", size >> 10); - /* - * set_pages_*() requires a global_flush_tlb() call after it. - * We do this after the printk so that if something went wrong in the - * change, the printk gets out at least to give a better debug hint - * of who is the culprit. - */ - global_flush_tlb(); - #ifdef CONFIG_CPA_DEBUG printk("Testing CPA: undo %lx-%lx\n", start, start + size); set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); - global_flush_tlb(); printk("Testing CPA: write protecting again\n"); set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); - global_flush_tlb(); #endif } #endif diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 4757be7b5e5..9b69fa54a83 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -610,22 +610,12 @@ void mark_rodata_ro(void) printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", (end - start) >> 10); - /* - * set_memory_*() requires a global_flush_tlb() call after it. - * We do this after the printk so that if something went wrong in the - * change, the printk gets out at least to give a better debug hint - * of who is the culprit. - */ - global_flush_tlb(); - #ifdef CONFIG_CPA_DEBUG printk("Testing CPA: undo %lx-%lx\n", start, end); set_memory_rw(start, (end-start) >> PAGE_SHIFT); - global_flush_tlb(); printk("Testing CPA: again\n"); set_memory_ro(start, (end-start) >> PAGE_SHIFT); - global_flush_tlb(); #endif } #endif diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index b86f66fa518..6a9a1418bc9 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -96,8 +96,6 @@ static int ioremap_change_attr(unsigned long paddr, unsigned long size, err = set_memory_wb(vaddr, nrpages); break; } - if (!err) - global_flush_tlb(); return err; } diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index e4d2b6930e6..a2d747c0614 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -22,6 +22,36 @@ within(unsigned long addr, unsigned long start, unsigned long end) return addr >= start && addr < end; } +/* + * Flushing functions + */ +void clflush_cache_range(void *addr, int size) +{ + int i; + + for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size) + clflush(addr+i); +} + +static void flush_kernel_map(void *arg) +{ + /* + * Flush all to work around Errata in early athlons regarding + * large page flushing. + */ + __flush_tlb_all(); + + if (boot_cpu_data.x86_model >= 4) + wbinvd(); +} + +static void global_flush_tlb(void) +{ + BUG_ON(irqs_disabled()); + + on_each_cpu(flush_kernel_map, NULL, 1, 1); +} + /* * Certain areas of memory on x86 require very specific protection flags, * for example the BIOS area or kernel text. Callers don't always get this @@ -328,149 +358,124 @@ static int change_page_attr_clear(unsigned long addr, int numpages, int set_memory_uc(unsigned long addr, int numpages) { - pgprot_t uncached; + int err; - pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT; - return change_page_attr_set(addr, numpages, uncached); + err = change_page_attr_set(addr, numpages, + __pgprot(_PAGE_PCD | _PAGE_PWT)); + global_flush_tlb(); + return err; } EXPORT_SYMBOL(set_memory_uc); int set_memory_wb(unsigned long addr, int numpages) { - pgprot_t uncached; + int err; - pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT; - return change_page_attr_clear(addr, numpages, uncached); + err = change_page_attr_clear(addr, numpages, + __pgprot(_PAGE_PCD | _PAGE_PWT)); + global_flush_tlb(); + return err; } EXPORT_SYMBOL(set_memory_wb); int set_memory_x(unsigned long addr, int numpages) { - pgprot_t nx; + int err; - pgprot_val(nx) = _PAGE_NX; - return change_page_attr_clear(addr, numpages, nx); + err = change_page_attr_clear(addr, numpages, + __pgprot(_PAGE_NX)); + global_flush_tlb(); + return err; } EXPORT_SYMBOL(set_memory_x); int set_memory_nx(unsigned long addr, int numpages) { - pgprot_t nx; + int err; - pgprot_val(nx) = _PAGE_NX; - return change_page_attr_set(addr, numpages, nx); + err = change_page_attr_set(addr, numpages, + __pgprot(_PAGE_NX)); + global_flush_tlb(); + return err; } EXPORT_SYMBOL(set_memory_nx); int set_memory_ro(unsigned long addr, int numpages) { - pgprot_t rw; + int err; - pgprot_val(rw) = _PAGE_RW; - return change_page_attr_clear(addr, numpages, rw); + err = change_page_attr_clear(addr, numpages, + __pgprot(_PAGE_RW)); + global_flush_tlb(); + return err; } int set_memory_rw(unsigned long addr, int numpages) { - pgprot_t rw; + int err; - pgprot_val(rw) = _PAGE_RW; - return change_page_attr_set(addr, numpages, rw); + err = change_page_attr_set(addr, numpages, + __pgprot(_PAGE_RW)); + global_flush_tlb(); + return err; } int set_memory_np(unsigned long addr, int numpages) { - pgprot_t present; + int err; - pgprot_val(present) = _PAGE_PRESENT; - return change_page_attr_clear(addr, numpages, present); + err = change_page_attr_clear(addr, numpages, + __pgprot(_PAGE_PRESENT)); + global_flush_tlb(); + return err; } int set_pages_uc(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t uncached; - pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT; - return change_page_attr_set(addr, numpages, uncached); + return set_memory_uc(addr, numpages); } EXPORT_SYMBOL(set_pages_uc); int set_pages_wb(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t uncached; - pgprot_val(uncached) = _PAGE_PCD | _PAGE_PWT; - return change_page_attr_clear(addr, numpages, uncached); + return set_memory_wb(addr, numpages); } EXPORT_SYMBOL(set_pages_wb); int set_pages_x(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t nx; - pgprot_val(nx) = _PAGE_NX; - return change_page_attr_clear(addr, numpages, nx); + return set_memory_x(addr, numpages); } EXPORT_SYMBOL(set_pages_x); int set_pages_nx(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t nx; - pgprot_val(nx) = _PAGE_NX; - return change_page_attr_set(addr, numpages, nx); + return set_memory_nx(addr, numpages); } EXPORT_SYMBOL(set_pages_nx); int set_pages_ro(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t rw; - pgprot_val(rw) = _PAGE_RW; - return change_page_attr_clear(addr, numpages, rw); + return set_memory_ro(addr, numpages); } int set_pages_rw(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t rw; - pgprot_val(rw) = _PAGE_RW; - return change_page_attr_set(addr, numpages, rw); + return set_memory_rw(addr, numpages); } -void clflush_cache_range(void *addr, int size) -{ - int i; - - for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size) - clflush(addr+i); -} - -static void flush_kernel_map(void *arg) -{ - /* - * Flush all to work around Errata in early athlons regarding - * large page flushing. - */ - __flush_tlb_all(); - - if (boot_cpu_data.x86_model >= 4) - wbinvd(); -} - -void global_flush_tlb(void) -{ - BUG_ON(irqs_disabled()); - - on_each_cpu(flush_kernel_map, NULL, 1, 1); -} -EXPORT_SYMBOL(global_flush_tlb); #ifdef CONFIG_DEBUG_PAGEALLOC diff --git a/drivers/char/agp/ali-agp.c b/drivers/char/agp/ali-agp.c index aa5ddb716ff..1ffb381130c 100644 --- a/drivers/char/agp/ali-agp.c +++ b/drivers/char/agp/ali-agp.c @@ -145,7 +145,6 @@ static void *m1541_alloc_page(struct agp_bridge_data *bridge) void *addr = agp_generic_alloc_page(agp_bridge); u32 temp; - global_flush_tlb(); if (!addr) return NULL; @@ -162,7 +161,6 @@ static void ali_destroy_page(void * addr, int flags) if (flags & AGP_PAGE_DESTROY_UNMAP) { global_cache_flush(); /* is this really needed? --hch */ agp_generic_destroy_page(addr, flags); - global_flush_tlb(); } else agp_generic_destroy_page(addr, flags); } diff --git a/drivers/char/agp/i460-agp.c b/drivers/char/agp/i460-agp.c index e72a83e2bad..76f581c85a7 100644 --- a/drivers/char/agp/i460-agp.c +++ b/drivers/char/agp/i460-agp.c @@ -527,7 +527,6 @@ static void *i460_alloc_page (struct agp_bridge_data *bridge) if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) { page = agp_generic_alloc_page(agp_bridge); - global_flush_tlb(); } else /* Returning NULL would cause problems */ /* AK: really dubious code. */ @@ -539,7 +538,6 @@ static void i460_destroy_page (void *page, int flags) { if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) { agp_generic_destroy_page(page, flags); - global_flush_tlb(); } } diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c index c03a7143928..189efb6ef97 100644 --- a/drivers/char/agp/intel-agp.c +++ b/drivers/char/agp/intel-agp.c @@ -212,11 +212,9 @@ static void *i8xx_alloc_pages(void) if (set_pages_uc(page, 4) < 0) { set_pages_wb(page, 4); - global_flush_tlb(); __free_pages(page, 2); return NULL; } - global_flush_tlb(); get_page(page); atomic_inc(&agp_bridge->current_memory_agp); return page_address(page); @@ -231,7 +229,6 @@ static void i8xx_destroy_pages(void *addr) page = virt_to_page(addr); set_pages_wb(page, 4); - global_flush_tlb(); put_page(page); __free_pages(page, 2); atomic_dec(&agp_bridge->current_memory_agp); @@ -341,7 +338,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type) switch (pg_count) { case 1: addr = agp_bridge->driver->agp_alloc_page(agp_bridge); - global_flush_tlb(); break; case 4: /* kludge to get 4 physical pages for ARGB cursor */ @@ -404,7 +400,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr) else { agp_bridge->driver->agp_destroy_page(gart_to_virt(curr->memory[0]), AGP_PAGE_DESTROY_UNMAP); - global_flush_tlb(); agp_bridge->driver->agp_destroy_page(gart_to_virt(curr->memory[0]), AGP_PAGE_DESTROY_FREE); } diff --git a/drivers/video/vermilion/vermilion.c b/drivers/video/vermilion/vermilion.c index fb72778dee4..1c656667b93 100644 --- a/drivers/video/vermilion/vermilion.c +++ b/drivers/video/vermilion/vermilion.c @@ -124,13 +124,8 @@ static int vmlfb_alloc_vram_area(struct vram_area *va, unsigned max_order, /* * Change caching policy of the linear kernel map to avoid * mapping type conflicts with user-space mappings. - * The first global_flush_tlb() is really only there to do a global - * wbinvd(). */ - - global_flush_tlb(); set_pages_uc(virt_to_page(va->logical), va->size >> PAGE_SHIFT); - global_flush_tlb(); printk(KERN_DEBUG MODULE_NAME ": Allocated %ld bytes vram area at 0x%08lx\n", @@ -156,7 +151,6 @@ static void vmlfb_free_vram_area(struct vram_area *va) set_pages_wb(virt_to_page(va->logical), va->size >> PAGE_SHIFT); - global_flush_tlb(); /* * Decrease the usage count on the pages we've used diff --git a/include/asm-x86/agp.h b/include/asm-x86/agp.h index f6df7256183..0c309b9a521 100644 --- a/include/asm-x86/agp.h +++ b/include/asm-x86/agp.h @@ -12,13 +12,9 @@ * page. This avoids data corruption on some CPUs. */ -/* - * Caller's responsibility to call global_flush_tlb() for performance - * reasons - */ #define map_page_into_agp(page) set_pages_uc(page, 1) #define unmap_page_from_agp(page) set_pages_wb(page, 1) -#define flush_agp_mappings() global_flush_tlb() +#define flush_agp_mappings() do { } while (0) /* * Could use CLFLUSH here if the cpu supports it. But then it would diff --git a/include/asm-x86/cacheflush.h b/include/asm-x86/cacheflush.h index d15ff359d3e..157da0206cc 100644 --- a/include/asm-x86/cacheflush.h +++ b/include/asm-x86/cacheflush.h @@ -24,7 +24,6 @@ #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ memcpy(dst, src, len) -void global_flush_tlb(void); int __deprecated_for_modules change_page_attr(struct page *page, int numpages, pgprot_t prot); diff --git a/sound/pci/intel8x0.c b/sound/pci/intel8x0.c index e5650905296..4bb97646a67 100644 --- a/sound/pci/intel8x0.c +++ b/sound/pci/intel8x0.c @@ -715,7 +715,6 @@ static void fill_nocache(void *buf, int size, int nocache) set_pages_uc(virt_to_page(buf), size); else set_pages_wb(virt_to_page(buf), size); - global_flush_tlb(); } #else #define fill_nocache(buf, size, nocache) do { ; } while (0)