2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* linux/arch/arm/mm/fault-armv.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
|
|
* Modifications for ARM processor (c) 1995-2002 Russell King
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/vmalloc.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
2008-09-05 13:08:44 +00:00
|
|
|
#include <asm/bugs.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/cacheflush.h>
|
2008-08-10 17:10:19 +00:00
|
|
|
#include <asm/cachetype.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2009-10-24 13:11:59 +00:00
|
|
|
#include "mm.h"
|
|
|
|
|
2008-09-06 19:04:59 +00:00
|
|
|
static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We take the easy way out of this problem - we make the
|
|
|
|
* PTE uncacheable. However, we leave the write buffer on.
|
[PATCH] mm: arm ready for split ptlock
Prepare arm for the split page_table_lock: three issues.
Signal handling's preserve and restore of iwmmxt context currently involves
reading and writing that context to and from user space, while holding
page_table_lock to secure the user page(s) against kswapd. If we split the
lock, then the structure might span two pages, secured by to read into and
write from a kernel stack buffer, copying that out and in without locking (the
structure is 160 bytes in size, and here we're near the top of the kernel
stack). Or would the overhead be noticeable?
arm_syscall's cmpxchg emulation use pte_offset_map_lock, instead of
pte_offset_map and mm-wide page_table_lock; and strictly, it should now also
take mmap_sem before descending to pmd, to guard against another thread
munmapping, and the page table pulled out beneath this thread.
Updated two comments in fault-armv.c. adjust_pte is interesting, since its
modification of a pte in one part of the mm depends on the lock held when
calling update_mmu_cache for a pte in some other part of that mm. This can't
be done with a split page_table_lock (and we've already taken the lowest lock
in the hierarchy here): so we'll have to disable split on arm, unless
CONFIG_CPU_CACHE_VIPT to ensures adjust_pte never used.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:36 +00:00
|
|
|
*
|
|
|
|
* Note that the pte lock held when calling update_mmu_cache must also
|
|
|
|
* guard the pte (somewhere else in the same mm) that we modify here.
|
|
|
|
* Therefore those configurations which might call adjust_pte (those
|
|
|
|
* without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-12-18 16:21:35 +00:00
|
|
|
static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
|
|
|
|
pte_t *ptep)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-12-18 16:21:35 +00:00
|
|
|
pte_t entry = *ptep;
|
2008-07-27 09:35:54 +00:00
|
|
|
int ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-27 09:35:54 +00:00
|
|
|
/*
|
|
|
|
* If this page is present, it's actually being shared.
|
|
|
|
*/
|
|
|
|
ret = pte_present(entry);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* If this page isn't present, or is already setup to
|
|
|
|
* fault (ie, is old), we can safely ignore any issues.
|
|
|
|
*/
|
2008-09-06 19:04:59 +00:00
|
|
|
if (ret && (pte_val(entry) & L_PTE_MT_MASK) != shared_pte_mask) {
|
2009-01-16 22:02:54 +00:00
|
|
|
unsigned long pfn = pte_pfn(entry);
|
|
|
|
flush_cache_page(vma, address, pfn);
|
|
|
|
outer_flush_range((pfn << PAGE_SHIFT),
|
|
|
|
(pfn << PAGE_SHIFT) + PAGE_SIZE);
|
2008-09-06 19:04:59 +00:00
|
|
|
pte_val(entry) &= ~L_PTE_MT_MASK;
|
|
|
|
pte_val(entry) |= shared_pte_mask;
|
2009-12-18 16:21:35 +00:00
|
|
|
set_pte_at(vma->vm_mm, address, ptep, entry);
|
2005-04-16 22:20:36 +00:00
|
|
|
flush_tlb_page(vma, address);
|
|
|
|
}
|
2009-12-18 16:21:35 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
pgd = pgd_offset(vma->vm_mm, address);
|
2009-12-18 16:23:44 +00:00
|
|
|
if (pgd_none_or_clear_bad(pgd))
|
|
|
|
return 0;
|
2009-12-18 16:21:35 +00:00
|
|
|
|
|
|
|
pmd = pmd_offset(pgd, address);
|
2009-12-18 16:23:44 +00:00
|
|
|
if (pmd_none_or_clear_bad(pmd))
|
|
|
|
return 0;
|
2009-12-18 16:21:35 +00:00
|
|
|
|
|
|
|
pte = pte_offset_map(pmd, address);
|
|
|
|
|
|
|
|
ret = do_adjust_pte(vma, address, pte);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
pte_unmap(pte);
|
2009-12-18 16:21:35 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2005-06-20 08:51:03 +00:00
|
|
|
make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
|
|
struct vm_area_struct *mpnt;
|
|
|
|
struct prio_tree_iter iter;
|
|
|
|
unsigned long offset;
|
|
|
|
pgoff_t pgoff;
|
|
|
|
int aliases = 0;
|
|
|
|
|
|
|
|
pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we have any shared mappings that are in the same mm
|
|
|
|
* space, then we need to handle them specially to maintain
|
|
|
|
* cache coherency.
|
|
|
|
*/
|
|
|
|
flush_dcache_mmap_lock(mapping);
|
|
|
|
vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) {
|
|
|
|
/*
|
|
|
|
* If this VMA is not in our MM, we can ignore it.
|
|
|
|
* Note that we intentionally mask out the VMA
|
|
|
|
* that we are fixing up.
|
|
|
|
*/
|
|
|
|
if (mpnt->vm_mm != mm || mpnt == vma)
|
|
|
|
continue;
|
|
|
|
if (!(mpnt->vm_flags & VM_MAYSHARE))
|
|
|
|
continue;
|
|
|
|
offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
|
|
|
|
aliases += adjust_pte(mpnt, mpnt->vm_start + offset);
|
|
|
|
}
|
|
|
|
flush_dcache_mmap_unlock(mapping);
|
|
|
|
if (aliases)
|
|
|
|
adjust_pte(vma, addr);
|
|
|
|
else
|
2005-06-20 08:51:03 +00:00
|
|
|
flush_cache_page(vma, addr, pfn);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Take care of architecture specific things when placing a new PTE into
|
|
|
|
* a page table, or changing an existing PTE. Basically, there are two
|
|
|
|
* things that we need to take care of:
|
|
|
|
*
|
|
|
|
* 1. If PG_dcache_dirty is set for the page, we need to ensure
|
|
|
|
* that any cache entries for the kernels virtual memory
|
|
|
|
* range are written back to the page.
|
|
|
|
* 2. If we have multiple shared mappings of the same space in
|
|
|
|
* an object, we need to deal with the cache aliasing issues.
|
|
|
|
*
|
[PATCH] mm: arm ready for split ptlock
Prepare arm for the split page_table_lock: three issues.
Signal handling's preserve and restore of iwmmxt context currently involves
reading and writing that context to and from user space, while holding
page_table_lock to secure the user page(s) against kswapd. If we split the
lock, then the structure might span two pages, secured by to read into and
write from a kernel stack buffer, copying that out and in without locking (the
structure is 160 bytes in size, and here we're near the top of the kernel
stack). Or would the overhead be noticeable?
arm_syscall's cmpxchg emulation use pte_offset_map_lock, instead of
pte_offset_map and mm-wide page_table_lock; and strictly, it should now also
take mmap_sem before descending to pmd, to guard against another thread
munmapping, and the page table pulled out beneath this thread.
Updated two comments in fault-armv.c. adjust_pte is interesting, since its
modification of a pte in one part of the mm depends on the lock held when
calling update_mmu_cache for a pte in some other part of that mm. This can't
be done with a split page_table_lock (and we've already taken the lowest lock
in the hierarchy here): so we'll have to disable split on arm, unless
CONFIG_CPU_CACHE_VIPT to ensures adjust_pte never used.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:36 +00:00
|
|
|
* Note that the pte lock will be held.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
|
|
|
|
{
|
|
|
|
unsigned long pfn = pte_pfn(pte);
|
2005-06-20 08:51:03 +00:00
|
|
|
struct address_space *mapping;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
if (!pfn_valid(pfn))
|
|
|
|
return;
|
2005-06-20 08:51:03 +00:00
|
|
|
|
2009-10-25 10:23:04 +00:00
|
|
|
/*
|
|
|
|
* The zero page is never written to, so never has any dirty
|
|
|
|
* cache lines, and therefore never needs to be flushed.
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
page = pfn_to_page(pfn);
|
2009-10-25 10:23:04 +00:00
|
|
|
if (page == ZERO_PAGE(0))
|
|
|
|
return;
|
|
|
|
|
2005-06-20 08:51:03 +00:00
|
|
|
mapping = page_mapping(page);
|
2008-06-13 09:28:36 +00:00
|
|
|
#ifndef CONFIG_SMP
|
2009-10-12 08:50:23 +00:00
|
|
|
if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
|
|
|
|
__flush_dcache_page(mapping, page);
|
2008-06-13 09:28:36 +00:00
|
|
|
#endif
|
2009-10-12 08:50:23 +00:00
|
|
|
if (mapping) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (cache_is_vivt())
|
2005-06-20 08:51:03 +00:00
|
|
|
make_coherent(mapping, vma, addr, pfn);
|
2008-06-13 09:28:36 +00:00
|
|
|
else if (vma->vm_flags & VM_EXEC)
|
|
|
|
__flush_icache_all();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check whether the write buffer has physical address aliasing
|
|
|
|
* issues. If it has, we need to avoid them for the case where
|
|
|
|
* we have several shared mappings of the same object in user
|
|
|
|
* space.
|
|
|
|
*/
|
|
|
|
static int __init check_writebuffer(unsigned long *p1, unsigned long *p2)
|
|
|
|
{
|
|
|
|
register unsigned long zero = 0, one = 1, val;
|
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
mb();
|
|
|
|
*p1 = one;
|
|
|
|
mb();
|
|
|
|
*p2 = zero;
|
|
|
|
mb();
|
|
|
|
val = *p1;
|
|
|
|
mb();
|
|
|
|
local_irq_enable();
|
|
|
|
return val != zero;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __init check_writebuffer_bugs(void)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
const char *reason;
|
|
|
|
unsigned long v = 1;
|
|
|
|
|
|
|
|
printk(KERN_INFO "CPU: Testing write buffer coherency: ");
|
|
|
|
|
|
|
|
page = alloc_page(GFP_KERNEL);
|
|
|
|
if (page) {
|
|
|
|
unsigned long *p1, *p2;
|
2009-12-23 19:54:31 +00:00
|
|
|
pgprot_t prot = __pgprot_modify(PAGE_KERNEL,
|
|
|
|
L_PTE_MT_MASK, L_PTE_MT_BUFFERABLE);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
p1 = vmap(&page, 1, VM_IOREMAP, prot);
|
|
|
|
p2 = vmap(&page, 1, VM_IOREMAP, prot);
|
|
|
|
|
|
|
|
if (p1 && p2) {
|
|
|
|
v = check_writebuffer(p1, p2);
|
|
|
|
reason = "enabling work-around";
|
|
|
|
} else {
|
|
|
|
reason = "unable to map memory\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
vunmap(p1);
|
|
|
|
vunmap(p2);
|
|
|
|
put_page(page);
|
|
|
|
} else {
|
|
|
|
reason = "unable to grab page\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
if (v) {
|
|
|
|
printk("failed, %s\n", reason);
|
2008-09-06 19:04:59 +00:00
|
|
|
shared_pte_mask = L_PTE_MT_UNCACHED;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
|
|
|
printk("ok\n");
|
|
|
|
}
|
|
|
|
}
|