mm: introduce new field "managed_pages" to struct zone
Currently a zone's present_pages is calcuated as below, which is inaccurate and may cause trouble to memory hotplug. spanned_pages - absent_pages - memmap_pages - dma_reserve. During fixing bugs caused by inaccurate zone->present_pages, we found zone->present_pages has been abused. The field zone->present_pages may have different meanings in different contexts: 1) pages existing in a zone. 2) pages managed by the buddy system. For more discussions about the issue, please refer to: http://lkml.org/lkml/2012/11/5/866 https://patchwork.kernel.org/patch/1346751/ This patchset tries to introduce a new field named "managed_pages" to struct zone, which counts "pages managed by the buddy system". And revert zone->present_pages to count "physical pages existing in a zone", which also keep in consistence with pgdat->node_present_pages. We will set an initial value for zone->managed_pages in function free_area_init_core() and will adjust it later if the initial value is inaccurate. For DMA/normal zones, the initial value is set to: (spanned_pages - absent_pages - memmap_pages - dma_reserve) Later zone->managed_pages will be adjusted to the accurate value when the bootmem allocator frees all free pages to the buddy system in function free_all_bootmem_node() and free_all_bootmem(). The bootmem allocator doesn't touch highmem pages, so highmem zones' managed_pages is set to the accurate value "spanned_pages - absent_pages" in function free_area_init_core() and won't be updated anymore. This patch also adds a new field "managed_pages" to /proc/zoneinfo and sysrq showmem. [akpm@linux-foundation.org: small comment tweaks] Signed-off-by: Jiang Liu <jiang.liu@huawei.com> Cc: Wen Congyang <wency@cn.fujitsu.com> Cc: David Rientjes <rientjes@google.com> Cc: Maciej Rutecki <maciej.rutecki@gmail.com> Tested-by: Chris Clayton <chris2553@googlemail.com> Cc: "Rafael J . Wysocki" <rjw@sisk.pl> Cc: Mel Gorman <mgorman@suse.de> Cc: Minchan Kim <minchan@kernel.org> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Michal Hocko <mhocko@suse.cz> Cc: Jianguo Wu <wujianguo@huawei.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
c2d23f919b
commit
9feedc9d83
|
@ -460,17 +460,44 @@ struct zone {
|
||||||
unsigned long zone_start_pfn;
|
unsigned long zone_start_pfn;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* zone_start_pfn, spanned_pages and present_pages are all
|
* spanned_pages is the total pages spanned by the zone, including
|
||||||
* protected by span_seqlock. It is a seqlock because it has
|
* holes, which is calculated as:
|
||||||
* to be read outside of zone->lock, and it is done in the main
|
* spanned_pages = zone_end_pfn - zone_start_pfn;
|
||||||
* allocator path. But, it is written quite infrequently.
|
|
||||||
*
|
*
|
||||||
* The lock is declared along with zone->lock because it is
|
* present_pages is physical pages existing within the zone, which
|
||||||
|
* is calculated as:
|
||||||
|
* present_pages = spanned_pages - absent_pages(pags in holes);
|
||||||
|
*
|
||||||
|
* managed_pages is present pages managed by the buddy system, which
|
||||||
|
* is calculated as (reserved_pages includes pages allocated by the
|
||||||
|
* bootmem allocator):
|
||||||
|
* managed_pages = present_pages - reserved_pages;
|
||||||
|
*
|
||||||
|
* So present_pages may be used by memory hotplug or memory power
|
||||||
|
* management logic to figure out unmanaged pages by checking
|
||||||
|
* (present_pages - managed_pages). And managed_pages should be used
|
||||||
|
* by page allocator and vm scanner to calculate all kinds of watermarks
|
||||||
|
* and thresholds.
|
||||||
|
*
|
||||||
|
* Locking rules:
|
||||||
|
*
|
||||||
|
* zone_start_pfn and spanned_pages are protected by span_seqlock.
|
||||||
|
* It is a seqlock because it has to be read outside of zone->lock,
|
||||||
|
* and it is done in the main allocator path. But, it is written
|
||||||
|
* quite infrequently.
|
||||||
|
*
|
||||||
|
* The span_seq lock is declared along with zone->lock because it is
|
||||||
* frequently read in proximity to zone->lock. It's good to
|
* frequently read in proximity to zone->lock. It's good to
|
||||||
* give them a chance of being in the same cacheline.
|
* give them a chance of being in the same cacheline.
|
||||||
|
*
|
||||||
|
* Write access to present_pages and managed_pages at runtime should
|
||||||
|
* be protected by lock_memory_hotplug()/unlock_memory_hotplug().
|
||||||
|
* Any reader who can't tolerant drift of present_pages and
|
||||||
|
* managed_pages should hold memory hotplug lock to get a stable value.
|
||||||
*/
|
*/
|
||||||
unsigned long spanned_pages; /* total size, including holes */
|
unsigned long spanned_pages;
|
||||||
unsigned long present_pages; /* amount of memory (excluding holes) */
|
unsigned long present_pages;
|
||||||
|
unsigned long managed_pages;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* rarely used fields:
|
* rarely used fields:
|
||||||
|
|
21
mm/bootmem.c
21
mm/bootmem.c
|
@ -229,6 +229,22 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void reset_node_lowmem_managed_pages(pg_data_t *pgdat)
|
||||||
|
{
|
||||||
|
struct zone *z;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* In free_area_init_core(), highmem zone's managed_pages is set to
|
||||||
|
* present_pages, and bootmem allocator doesn't allocate from highmem
|
||||||
|
* zones. So there's no need to recalculate managed_pages because all
|
||||||
|
* highmem pages will be managed by the buddy system. Here highmem
|
||||||
|
* zone also includes highmem movable zone.
|
||||||
|
*/
|
||||||
|
for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
|
||||||
|
if (!is_highmem(z))
|
||||||
|
z->managed_pages = 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* free_all_bootmem_node - release a node's free pages to the buddy allocator
|
* free_all_bootmem_node - release a node's free pages to the buddy allocator
|
||||||
* @pgdat: node to be released
|
* @pgdat: node to be released
|
||||||
|
@ -238,6 +254,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
|
||||||
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
|
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
|
||||||
{
|
{
|
||||||
register_page_bootmem_info_node(pgdat);
|
register_page_bootmem_info_node(pgdat);
|
||||||
|
reset_node_lowmem_managed_pages(pgdat);
|
||||||
return free_all_bootmem_core(pgdat->bdata);
|
return free_all_bootmem_core(pgdat->bdata);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -250,6 +267,10 @@ unsigned long __init free_all_bootmem(void)
|
||||||
{
|
{
|
||||||
unsigned long total_pages = 0;
|
unsigned long total_pages = 0;
|
||||||
bootmem_data_t *bdata;
|
bootmem_data_t *bdata;
|
||||||
|
struct pglist_data *pgdat;
|
||||||
|
|
||||||
|
for_each_online_pgdat(pgdat)
|
||||||
|
reset_node_lowmem_managed_pages(pgdat);
|
||||||
|
|
||||||
list_for_each_entry(bdata, &bdata_list, list)
|
list_for_each_entry(bdata, &bdata_list, list)
|
||||||
total_pages += free_all_bootmem_core(bdata);
|
total_pages += free_all_bootmem_core(bdata);
|
||||||
|
|
|
@ -106,6 +106,7 @@ static void get_page_bootmem(unsigned long info, struct page *page,
|
||||||
void __ref put_page_bootmem(struct page *page)
|
void __ref put_page_bootmem(struct page *page)
|
||||||
{
|
{
|
||||||
unsigned long type;
|
unsigned long type;
|
||||||
|
static DEFINE_MUTEX(ppb_lock);
|
||||||
|
|
||||||
type = (unsigned long) page->lru.next;
|
type = (unsigned long) page->lru.next;
|
||||||
BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
|
BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
|
||||||
|
@ -115,7 +116,14 @@ void __ref put_page_bootmem(struct page *page)
|
||||||
ClearPagePrivate(page);
|
ClearPagePrivate(page);
|
||||||
set_page_private(page, 0);
|
set_page_private(page, 0);
|
||||||
INIT_LIST_HEAD(&page->lru);
|
INIT_LIST_HEAD(&page->lru);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Please refer to comment for __free_pages_bootmem()
|
||||||
|
* for why we serialize here.
|
||||||
|
*/
|
||||||
|
mutex_lock(&ppb_lock);
|
||||||
__free_pages_bootmem(page, 0);
|
__free_pages_bootmem(page, 0);
|
||||||
|
mutex_unlock(&ppb_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -748,6 +756,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
zone->managed_pages += onlined_pages;
|
||||||
zone->present_pages += onlined_pages;
|
zone->present_pages += onlined_pages;
|
||||||
zone->zone_pgdat->node_present_pages += onlined_pages;
|
zone->zone_pgdat->node_present_pages += onlined_pages;
|
||||||
if (onlined_pages) {
|
if (onlined_pages) {
|
||||||
|
@ -1321,6 +1330,7 @@ repeat:
|
||||||
/* reset pagetype flags and makes migrate type to be MOVABLE */
|
/* reset pagetype flags and makes migrate type to be MOVABLE */
|
||||||
undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
|
undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
|
||||||
/* removal success */
|
/* removal success */
|
||||||
|
zone->managed_pages -= offlined_pages;
|
||||||
zone->present_pages -= offlined_pages;
|
zone->present_pages -= offlined_pages;
|
||||||
zone->zone_pgdat->node_present_pages -= offlined_pages;
|
zone->zone_pgdat->node_present_pages -= offlined_pages;
|
||||||
totalram_pages -= offlined_pages;
|
totalram_pages -= offlined_pages;
|
||||||
|
|
|
@ -137,6 +137,22 @@ unsigned long __init free_low_memory_core_early(int nodeid)
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void reset_node_lowmem_managed_pages(pg_data_t *pgdat)
|
||||||
|
{
|
||||||
|
struct zone *z;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* In free_area_init_core(), highmem zone's managed_pages is set to
|
||||||
|
* present_pages, and bootmem allocator doesn't allocate from highmem
|
||||||
|
* zones. So there's no need to recalculate managed_pages because all
|
||||||
|
* highmem pages will be managed by the buddy system. Here highmem
|
||||||
|
* zone also includes highmem movable zone.
|
||||||
|
*/
|
||||||
|
for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
|
||||||
|
if (!is_highmem(z))
|
||||||
|
z->managed_pages = 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* free_all_bootmem_node - release a node's free pages to the buddy allocator
|
* free_all_bootmem_node - release a node's free pages to the buddy allocator
|
||||||
* @pgdat: node to be released
|
* @pgdat: node to be released
|
||||||
|
@ -146,6 +162,7 @@ unsigned long __init free_low_memory_core_early(int nodeid)
|
||||||
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
|
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
|
||||||
{
|
{
|
||||||
register_page_bootmem_info_node(pgdat);
|
register_page_bootmem_info_node(pgdat);
|
||||||
|
reset_node_lowmem_managed_pages(pgdat);
|
||||||
|
|
||||||
/* free_low_memory_core_early(MAX_NUMNODES) will be called later */
|
/* free_low_memory_core_early(MAX_NUMNODES) will be called later */
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -158,6 +175,11 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
|
||||||
*/
|
*/
|
||||||
unsigned long __init free_all_bootmem(void)
|
unsigned long __init free_all_bootmem(void)
|
||||||
{
|
{
|
||||||
|
struct pglist_data *pgdat;
|
||||||
|
|
||||||
|
for_each_online_pgdat(pgdat)
|
||||||
|
reset_node_lowmem_managed_pages(pgdat);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
|
* We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
|
||||||
* because in some case like Node0 doesn't have RAM installed
|
* because in some case like Node0 doesn't have RAM installed
|
||||||
|
|
|
@ -735,6 +735,13 @@ static void __free_pages_ok(struct page *page, unsigned int order)
|
||||||
local_irq_restore(flags);
|
local_irq_restore(flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Read access to zone->managed_pages is safe because it's unsigned long,
|
||||||
|
* but we still need to serialize writers. Currently all callers of
|
||||||
|
* __free_pages_bootmem() except put_page_bootmem() should only be used
|
||||||
|
* at boot time. So for shorter boot time, we shift the burden to
|
||||||
|
* put_page_bootmem() to serialize writers.
|
||||||
|
*/
|
||||||
void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
|
void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
|
||||||
{
|
{
|
||||||
unsigned int nr_pages = 1 << order;
|
unsigned int nr_pages = 1 << order;
|
||||||
|
@ -750,6 +757,7 @@ void __meminit __free_pages_bootmem(struct page *page, unsigned int order)
|
||||||
set_page_count(p, 0);
|
set_page_count(p, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
page_zone(page)->managed_pages += 1 << order;
|
||||||
set_page_refcounted(page);
|
set_page_refcounted(page);
|
||||||
__free_pages(page, order);
|
__free_pages(page, order);
|
||||||
}
|
}
|
||||||
|
@ -2984,6 +2992,7 @@ void show_free_areas(unsigned int filter)
|
||||||
" isolated(anon):%lukB"
|
" isolated(anon):%lukB"
|
||||||
" isolated(file):%lukB"
|
" isolated(file):%lukB"
|
||||||
" present:%lukB"
|
" present:%lukB"
|
||||||
|
" managed:%lukB"
|
||||||
" mlocked:%lukB"
|
" mlocked:%lukB"
|
||||||
" dirty:%lukB"
|
" dirty:%lukB"
|
||||||
" writeback:%lukB"
|
" writeback:%lukB"
|
||||||
|
@ -3013,6 +3022,7 @@ void show_free_areas(unsigned int filter)
|
||||||
K(zone_page_state(zone, NR_ISOLATED_ANON)),
|
K(zone_page_state(zone, NR_ISOLATED_ANON)),
|
||||||
K(zone_page_state(zone, NR_ISOLATED_FILE)),
|
K(zone_page_state(zone, NR_ISOLATED_FILE)),
|
||||||
K(zone->present_pages),
|
K(zone->present_pages),
|
||||||
|
K(zone->managed_pages),
|
||||||
K(zone_page_state(zone, NR_MLOCK)),
|
K(zone_page_state(zone, NR_MLOCK)),
|
||||||
K(zone_page_state(zone, NR_FILE_DIRTY)),
|
K(zone_page_state(zone, NR_FILE_DIRTY)),
|
||||||
K(zone_page_state(zone, NR_WRITEBACK)),
|
K(zone_page_state(zone, NR_WRITEBACK)),
|
||||||
|
@ -4502,48 +4512,54 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
|
||||||
|
|
||||||
for (j = 0; j < MAX_NR_ZONES; j++) {
|
for (j = 0; j < MAX_NR_ZONES; j++) {
|
||||||
struct zone *zone = pgdat->node_zones + j;
|
struct zone *zone = pgdat->node_zones + j;
|
||||||
unsigned long size, realsize, memmap_pages;
|
unsigned long size, realsize, freesize, memmap_pages;
|
||||||
|
|
||||||
size = zone_spanned_pages_in_node(nid, j, zones_size);
|
size = zone_spanned_pages_in_node(nid, j, zones_size);
|
||||||
realsize = size - zone_absent_pages_in_node(nid, j,
|
realsize = freesize = size - zone_absent_pages_in_node(nid, j,
|
||||||
zholes_size);
|
zholes_size);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Adjust realsize so that it accounts for how much memory
|
* Adjust freesize so that it accounts for how much memory
|
||||||
* is used by this zone for memmap. This affects the watermark
|
* is used by this zone for memmap. This affects the watermark
|
||||||
* and per-cpu initialisations
|
* and per-cpu initialisations
|
||||||
*/
|
*/
|
||||||
memmap_pages =
|
memmap_pages =
|
||||||
PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
|
PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
|
||||||
if (realsize >= memmap_pages) {
|
if (freesize >= memmap_pages) {
|
||||||
realsize -= memmap_pages;
|
freesize -= memmap_pages;
|
||||||
if (memmap_pages)
|
if (memmap_pages)
|
||||||
printk(KERN_DEBUG
|
printk(KERN_DEBUG
|
||||||
" %s zone: %lu pages used for memmap\n",
|
" %s zone: %lu pages used for memmap\n",
|
||||||
zone_names[j], memmap_pages);
|
zone_names[j], memmap_pages);
|
||||||
} else
|
} else
|
||||||
printk(KERN_WARNING
|
printk(KERN_WARNING
|
||||||
" %s zone: %lu pages exceeds realsize %lu\n",
|
" %s zone: %lu pages exceeds freesize %lu\n",
|
||||||
zone_names[j], memmap_pages, realsize);
|
zone_names[j], memmap_pages, freesize);
|
||||||
|
|
||||||
/* Account for reserved pages */
|
/* Account for reserved pages */
|
||||||
if (j == 0 && realsize > dma_reserve) {
|
if (j == 0 && freesize > dma_reserve) {
|
||||||
realsize -= dma_reserve;
|
freesize -= dma_reserve;
|
||||||
printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
|
printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
|
||||||
zone_names[0], dma_reserve);
|
zone_names[0], dma_reserve);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!is_highmem_idx(j))
|
if (!is_highmem_idx(j))
|
||||||
nr_kernel_pages += realsize;
|
nr_kernel_pages += freesize;
|
||||||
nr_all_pages += realsize;
|
nr_all_pages += freesize;
|
||||||
|
|
||||||
zone->spanned_pages = size;
|
zone->spanned_pages = size;
|
||||||
zone->present_pages = realsize;
|
zone->present_pages = freesize;
|
||||||
|
/*
|
||||||
|
* Set an approximate value for lowmem here, it will be adjusted
|
||||||
|
* when the bootmem allocator frees pages into the buddy system.
|
||||||
|
* And all highmem pages will be managed by the buddy system.
|
||||||
|
*/
|
||||||
|
zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
|
||||||
#ifdef CONFIG_NUMA
|
#ifdef CONFIG_NUMA
|
||||||
zone->node = nid;
|
zone->node = nid;
|
||||||
zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
|
zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio)
|
||||||
/ 100;
|
/ 100;
|
||||||
zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
|
zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
|
||||||
#endif
|
#endif
|
||||||
zone->name = zone_names[j];
|
zone->name = zone_names[j];
|
||||||
spin_lock_init(&zone->lock);
|
spin_lock_init(&zone->lock);
|
||||||
|
|
|
@ -994,14 +994,16 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
|
||||||
"\n high %lu"
|
"\n high %lu"
|
||||||
"\n scanned %lu"
|
"\n scanned %lu"
|
||||||
"\n spanned %lu"
|
"\n spanned %lu"
|
||||||
"\n present %lu",
|
"\n present %lu"
|
||||||
|
"\n managed %lu",
|
||||||
zone_page_state(zone, NR_FREE_PAGES),
|
zone_page_state(zone, NR_FREE_PAGES),
|
||||||
min_wmark_pages(zone),
|
min_wmark_pages(zone),
|
||||||
low_wmark_pages(zone),
|
low_wmark_pages(zone),
|
||||||
high_wmark_pages(zone),
|
high_wmark_pages(zone),
|
||||||
zone->pages_scanned,
|
zone->pages_scanned,
|
||||||
zone->spanned_pages,
|
zone->spanned_pages,
|
||||||
zone->present_pages);
|
zone->present_pages,
|
||||||
|
zone->managed_pages);
|
||||||
|
|
||||||
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
|
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
|
||||||
seq_printf(m, "\n %-12s %lu", vmstat_text[i],
|
seq_printf(m, "\n %-12s %lu", vmstat_text[i],
|
||||||
|
|
Reference in New Issue