/*

* linux/mm/page_alloc.c

*

* Manages the free list, the system allocates free pages here.

* Note that kmalloc() lives in slab.c

*

* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds

* Swap reorganised 29.12.95, Stephen Tweedie

* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999

* Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999

* Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999

* Zone balancing, Kanoj Sarcar, SGI, Jan 2000

* Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002

* (lots of bits borrowed from Ingo Molnar & Andrew Morton)

*/

#include <linux/stddef.h>

#include <linux/mm.h>

#include <linux/swap.h>

#include <linux/interrupt.h>

#include <linux/pagemap.h>

#include <linux/jiffies.h>

#include <linux/bootmem.h>

#include <linux/memblock.h>

#include <linux/compiler.h>

#include <linux/kernel.h>

#include <linux/kmemcheck.h>

#include <linux/module.h>

#include <linux/suspend.h>

#include <linux/pagevec.h>

#include <linux/blkdev.h>

#include <linux/slab.h>

#include <linux/ratelimit.h>

#include <linux/oom.h>

#include <linux/notifier.h>

#include <linux/topology.h>

#include <linux/sysctl.h>

#include <linux/cpu.h>

#include <linux/cpuset.h>

#include <linux/memory_hotplug.h>

#include <linux/nodemask.h>

#include <linux/vmalloc.h>

#include <linux/vmstat.h>

#include <linux/mempolicy.h>

#include <linux/stop_machine.h>

#include <linux/sort.h>

#include <linux/pfn.h>

#include <linux/backing-dev.h>

#include <linux/fault-inject.h>

#include <linux/page-isolation.h>

#include <linux/page_cgroup.h>

#include <linux/debugobjects.h>

#include <linux/kmemleak.h>

#include <linux/compaction.h>

#include <trace/events/kmem.h>

#include <linux/ftrace_event.h>

#include <linux/memcontrol.h>

#include <linux/prefetch.h>

#include <linux/migrate.h>

#include <linux/page-debug-flags.h>

#include <asm/tlbflush.h>

#include <asm/div64.h>

#include "internal.h"

#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID

DEFINE_PER_CPU(int, numa_node);

EXPORT_PER_CPU_SYMBOL(numa_node);

#endif

#ifdef CONFIG_HAVE_MEMORYLESS_NODES

/*

* N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.

* It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.

* Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()

* defined in <linux/topology.h>.

*/

DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */

EXPORT_PER_CPU_SYMBOL(_numa_mem_);

#endif

/*

* Array of node states.

*/

nodemask_t node_states[NR_NODE_STATES] __read_mostly = {

[N_POSSIBLE] = NODE_MASK_ALL,

[N_ONLINE] = { { [0] = 1UL } },

#ifndef CONFIG_NUMA

[N_NORMAL_MEMORY] = { { [0] = 1UL } },

#ifdef CONFIG_HIGHMEM

[N_HIGH_MEMORY] = { { [0] = 1UL } },

#endif

[N_CPU] = { { [0] = 1UL } },

#endif /* NUMA */

};

EXPORT_SYMBOL(node_states);

unsigned long totalram_pages __read_mostly;

unsigned long totalreserve_pages __read_mostly;

/*

* When calculating the number of globally allowed dirty pages, there

* is a certain number of per-zone reserves that should not be

* considered dirtyable memory. This is the sum of those reserves

* over all existing zones that contribute dirtyable memory.

*/

unsigned long dirty_balance_reserve __read_mostly;

int percpu_pagelist_fraction;

gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;

#ifdef CONFIG_PM_SLEEP

/*

* The following functions are used by the suspend/hibernate code to temporarily

* change gfp_allowed_mask in order to avoid using I/O during memory allocations

* while devices are suspended. To avoid races with the suspend/hibernate code,

* they should always be called with pm_mutex held (gfp_allowed_mask also should

* only be modified with pm_mutex held, unless the suspend/hibernate code is

* guaranteed not to run in parallel with that modification).

*/

static gfp_t saved_gfp_mask;

void pm_restore_gfp_mask(void)

{

WARN_ON(!mutex_is_locked(&pm_mutex));

if (saved_gfp_mask) {

gfp_allowed_mask = saved_gfp_mask;

saved_gfp_mask = 0;

}

}

void pm_restrict_gfp_mask(void)

{

WARN_ON(!mutex_is_locked(&pm_mutex));

WARN_ON(saved_gfp_mask);

saved_gfp_mask = gfp_allowed_mask;

gfp_allowed_mask &= ~GFP_IOFS;

}

bool pm_suspended_storage(void)

{

if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)

return false;

return true;

}

#endif /* CONFIG_PM_SLEEP */

#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE

int pageblock_order __read_mostly;

#endif

static void __free_pages_ok(struct page *page, unsigned int order);

/*

* results with 256, 32 in the lowmem_reserve sysctl:

* 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)

* 1G machine -> (16M dma, 784M normal, 224M high)

* NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA

* HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL

* HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA

*

* TBD: should special case ZONE_DMA32 machines here - in those we normally

* don't need any ZONE_NORMAL reservation

*/

int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {

#ifdef CONFIG_ZONE_DMA

256,

#endif

#ifdef CONFIG_ZONE_DMA32

256,

#endif

#ifdef CONFIG_HIGHMEM

32,

#endif

32,

};

EXPORT_SYMBOL(totalram_pages);

static char * const zone_names[MAX_NR_ZONES] = {

#ifdef CONFIG_ZONE_DMA

"DMA",

#endif

#ifdef CONFIG_ZONE_DMA32

"DMA32",

#endif

"Normal",

#ifdef CONFIG_HIGHMEM

"HighMem",

#endif

"Movable",

};

int min_free_kbytes = 1024;

static unsigned long __meminitdata nr_kernel_pages;

static unsigned long __meminitdata nr_all_pages;

static unsigned long __meminitdata dma_reserve;

#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP

static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];

static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];

static unsigned long __initdata required_kernelcore;

static unsigned long __initdata required_movablecore;

static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */

int movable_zone;

EXPORT_SYMBOL(movable_zone);

#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */

#if MAX_NUMNODES > 1

int nr_node_ids __read_mostly = MAX_NUMNODES;

int nr_online_nodes __read_mostly = 1;

EXPORT_SYMBOL(nr_node_ids);

EXPORT_SYMBOL(nr_online_nodes);

#endif

int page_group_by_mobility_disabled __read_mostly;

/*

* NOTE:

* Don't use set_pageblock_migratetype(page, MIGRATE_ISOLATE) directly.

* Instead, use {un}set_pageblock_isolate.

*/

void set_pageblock_migratetype(struct page *page, int migratetype)

{

if (unlikely(page_group_by_mobility_disabled))

migratetype = MIGRATE_UNMOVABLE;

set_pageblock_flags_group(page, (unsigned long)migratetype,

PB_migrate, PB_migrate_end);

}

bool oom_killer_disabled __read_mostly;

#ifdef CONFIG_DEBUG_VM

static int page_outside_zone_boundaries(struct zone *zone, struct page *page)

{

int ret = 0;

unsigned seq;

unsigned long pfn = page_to_pfn(page);

do {

seq = zone_span_seqbegin(zone);

if (pfn >= zone->zone_start_pfn + zone->spanned_pages)

ret = 1;

else if (pfn < zone->zone_start_pfn)

ret = 1;

} while (zone_span_seqretry(zone, seq));

return ret;

}

static int page_is_consistent(struct zone *zone, struct page *page)

{

if (!pfn_valid_within(page_to_pfn(page)))

return 0;

if (zone != page_zone(page))

return 0;

return 1;

}

/*

* Temporary debugging check for pages not lying within a given zone.

*/

static int bad_range(struct zone *zone, struct page *page)

{

if (page_outside_zone_boundaries(zone, page))

return 1;

if (!page_is_consistent(zone, page))

return 1;

return 0;

}

#else

static inline int bad_range(struct zone *zone, struct page *page)

{

return 0;

}

#endif

static void bad_page(struct page *page)

{

static unsigned long resume;

static unsigned long nr_shown;

static unsigned long nr_unshown;

/* Don't complain about poisoned pages */

if (PageHWPoison(page)) {

reset_page_mapcount(page); /* remove PageBuddy */

return;

}

/*

* Allow a burst of 60 reports, then keep quiet for that minute;

* or allow a steady drip of one report per second.

*/

if (nr_shown == 60) {

if (time_before(jiffies, resume)) {

nr_unshown++;

goto out;

}

if (nr_unshown) {

printk(KERN_ALERT

"BUG: Bad page state: %lu messages suppressed\n",

nr_unshown);

nr_unshown = 0;

}

nr_shown = 0;

}

if (nr_shown++ == 0)

resume = jiffies + 60 * HZ;

printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",

current->comm, page_to_pfn(page));

dump_page(page);

print_modules();

dump_stack();

out:

/* Leave bad fields for debug, except PageBuddy could make trouble */

reset_page_mapcount(page); /* remove PageBuddy */

add_taint(TAINT_BAD_PAGE);

}

/*

* Higher-order pages are called "compound pages". They are structured thusly:

*

* The first PAGE_SIZE page is called the "head page".

*

* The remaining PAGE_SIZE pages are called "tail pages".

*

* All pages have PG_compound set. All tail pages have their ->first_page

* pointing at the head page.

*

* The first tail page's ->lru.next holds the address of the compound page's

* put_page() function. Its ->lru.prev holds the order of allocation.

* This usage means that zero-order pages may not be compound.

*/

static void free_compound_page(struct page *page)

{

__free_pages_ok(page, compound_order(page));

}

void prep_compound_page(struct page *page, unsigned long order)

{

int i;

int nr_pages = 1 << order;

set_compound_page_dtor(page, free_compound_page);

set_compound_order(page, order);

__SetPageHead(page);

for (i = 1; i < nr_pages; i++) {

struct page *p = page + i;

__SetPageTail(p);

set_page_count(p, 0);

p->first_page = page;

}

}

/* update __split_huge_page_refcount if you change this function */

static int destroy_compound_page(struct page *page, unsigned long order)

{

int i;

int nr_pages = 1 << order;

int bad = 0;

if (unlikely(compound_order(page) != order) ||

unlikely(!PageHead(page))) {

bad_page(page);

bad++;

}

__ClearPageHead(page);

for (i = 1; i < nr_pages; i++) {

struct page *p = page + i;

if (unlikely(!PageTail(p) || (p->first_page != page))) {

bad_page(page);

bad++;

}

__ClearPageTail(p);

}

return bad;

}

static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)

{

int i;

/*

* clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO

* and __GFP_HIGHMEM from hard or soft interrupt context.

*/

VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());

for (i = 0; i < (1 << order); i++)

clear_highpage(page + i);

}

#ifdef CONFIG_DEBUG_PAGEALLOC

unsigned int _debug_guardpage_minorder;

static int __init debug_guardpage_minorder_setup(char *buf)

{

unsigned long res;

if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {

printk(KERN_ERR "Bad debug_guardpage_minorder value\n");

return 0;

}

_debug_guardpage_minorder = res;

printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);

return 0;

}

__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);

static inline void set_page_guard_flag(struct page *page)

{

__set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);

}

static inline void clear_page_guard_flag(struct page *page)

{

__clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);

}

#else

static inline void set_page_guard_flag(struct page *page) { }

static inline void clear_page_guard_flag(struct page *page) { }

#endif

static inline void set_page_order(struct page *page, int order)

{

set_page_private(page, order);

__SetPageBuddy(page);

}

static inline void rmv_page_order(struct page *page)

{

__ClearPageBuddy(page);

set_page_private(page, 0);

}

/*

* Locate the struct page for both the matching buddy in our

* pair (buddy1) and the combined O(n+1) page they form (page).

*

* 1) Any buddy B1 will have an order O twin B2 which satisfies

* the following equation:

* B2 = B1 ^ (1 << O)

* For example, if the starting buddy (buddy2) is #8 its order

* 1 buddy is #10:

* B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10

*

* 2) Any buddy B will have an order O+1 parent P which

* satisfies the following equation:

* P = B & ~(1 << O)

*

* Assumption: *_mem_map is contiguous at least up to MAX_ORDER

*/

static inline unsigned long

__find_buddy_index(unsigned long page_idx, unsigned int order)

{

return page_idx ^ (1 << order);

}

/*

* This function checks whether a page is free && is the buddy

* we can do coalesce a page and its buddy if

* (a) the buddy is not in a hole &&

* (b) the buddy is in the buddy system &&

* (c) a page and its buddy have the same order &&

* (d) a page and its buddy are in the same zone.

*

* For recording whether a page is in the buddy system, we set ->_mapcount -2.

* Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.

*

* For recording page's order, we use page_private(page).

*/

static inline int page_is_buddy(struct page *page, struct page *buddy,

int order)

{

if (!pfn_valid_within(page_to_pfn(buddy)))

return 0;

if (page_zone_id(page) != page_zone_id(buddy))

return 0;

if (page_is_guard(buddy) && page_order(buddy) == order) {

VM_BUG_ON(page_count(buddy) != 0);

return 1;

}

if (PageBuddy(buddy) && page_order(buddy) == order) {

VM_BUG_ON(page_count(buddy) != 0);

return 1;

}

return 0;

}

/*

* Freeing function for a buddy system allocator.

*

* The concept of a buddy system is to maintain direct-mapped table

* (containing bit values) for memory blocks of various "orders".

* The bottom level table contains the map for the smallest allocatable

* units of memory (here, pages), and each level above it describes

* pairs of units from the levels below, hence, "buddies".

* At a high level, all that happens here is marking the table entry

* at the bottom level available, and propagating the changes upward

* as necessary, plus some accounting needed to play nicely with other

* parts of the VM system.

* At each level, we keep a list of pages, which are heads of continuous

* free pages of length of (1 << order) and marked with _mapcount -2. Page's

* order is recorded in page_private(page) field.

* So when we are allocating or freeing one, we can derive the state of the

* other. That is, if we allocate a small block, and both were

* free, the remainder of the region must be split into blocks.

* If a block is freed, and its buddy is also free, then this

* triggers coalescing into a block of larger size.

*

* -- wli

*/

static inline void __free_one_page(struct page *page,

struct zone *zone, unsigned int order,

int migratetype)

{

unsigned long page_idx;

unsigned long combined_idx;

unsigned long uninitialized_var(buddy_idx);

struct page *buddy;

if (unlikely(PageCompound(page)))

if (unlikely(destroy_compound_page(page, order)))

return;

VM_BUG_ON(migratetype == -1);

page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);

VM_BUG_ON(page_idx & ((1 << order) - 1));

VM_BUG_ON(bad_range(zone, page));

while (order < MAX_ORDER-1) {

buddy_idx = __find_buddy_index(page_idx, order);

buddy = page + (buddy_idx - page_idx);

if (!page_is_buddy(page, buddy, order))

break;

/*

* Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,

* merge with it and move up one order.

*/

if (page_is_guard(buddy)) {

clear_page_guard_flag(buddy);

set_page_private(page, 0);

__mod_zone_freepage_state(zone, 1 << order,

migratetype);

} else {

list_del(&buddy->lru);

zone->free_area[order].nr_free--;

rmv_page_order(buddy);

}

combined_idx = buddy_idx & page_idx;

page = page + (combined_idx - page_idx);

page_idx = combined_idx;

order++;

}

set_page_order(page, order);

/*

* If this is not the largest possible page, check if the buddy

* of the next-highest order is free. If it is, it's possible

* that pages are being freed that will coalesce soon. In case,

* that is happening, add the free page to the tail of the list

* so it's less likely to be used soon and more likely to be merged

* as a higher order page

*/

if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {

struct page *higher_page, *higher_buddy;

combined_idx = buddy_idx & page_idx;

higher_page = page + (combined_idx - page_idx);

buddy_idx = __find_buddy_index(combined_idx, order + 1);

higher_buddy = higher_page + (buddy_idx - combined_idx);

if (page_is_buddy(higher_page, higher_buddy, order + 1)) {

list_add_tail(&page->lru,

&zone->free_area[order].free_list[migratetype]);

goto out;

}

}

list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);

out:

zone->free_area[order].nr_free++;

}

static inline int free_pages_check(struct page *page)

{

if (unlikely(page_mapcount(page) |

(page->mapping != NULL) |

(atomic_read(&page->_count) != 0) |

(page->flags & PAGE_FLAGS_CHECK_AT_FREE) |

(mem_cgroup_bad_page_check(page)))) {

bad_page(page);

return 1;

}

if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)

page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;

return 0;

}

/*

* Frees a number of pages from the PCP lists

* Assumes all pages on list are in same zone, and of same order.

* count is the number of pages to free.

*

* If the zone was previously in an "all pages pinned" state then look to

* see if this freeing clears that state.

*

* And clear the zone's pages_scanned counter, to hold off the "all pages are

* pinned" detection logic.

*/

static void free_pcppages_bulk(struct zone *zone, int count,

struct per_cpu_pages *pcp)

{

int migratetype = 0;

int batch_free = 0;

int to_free = count;

spin_lock(&zone->lock);

zone->all_unreclaimable = 0;

zone->pages_scanned = 0;

while (to_free) {

struct page *page;

struct list_head *list;

/*

* Remove pages from lists in a round-robin fashion. A

* batch_free count is maintained that is incremented when an

* empty list is encountered. This is so more pages are freed

* off fuller lists instead of spinning excessively around empty

* lists

*/

do {

batch_free++;

if (++migratetype == MIGRATE_PCPTYPES)

migratetype = 0;

list = &pcp->lists[migratetype];

} while (list_empty(list));

/* This is the only non-empty list. Free them all. */

if (batch_free == MIGRATE_PCPTYPES)

batch_free = to_free;

do {

int mt; /* migratetype of the to-be-freed page */

page = list_entry(list->prev, struct page, lru);

/* must delete as __free_one_page list manipulates */

list_del(&page->lru);

mt = get_freepage_migratetype(page);

/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */

__free_one_page(page, zone, 0, mt);

trace_mm_page_pcpu_drain(page, 0, mt);

if (is_migrate_cma(mt))

__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);

} while (--to_free && --batch_free && !list_empty(list));

}

__mod_zone_page_state(zone, NR_FREE_PAGES, count);

spin_unlock(&zone->lock);

}

static void free_one_page(struct zone *zone, struct page *page, int order,

int migratetype)

{

spin_lock(&zone->lock);

zone->all_unreclaimable = 0;

zone->pages_scanned = 0;

__free_one_page(page, zone, order, migratetype);

if (unlikely(migratetype != MIGRATE_ISOLATE))

__mod_zone_freepage_state(zone, 1 << order, migratetype);

spin_unlock(&zone->lock);

}

static bool free_pages_prepare(struct page *page, unsigned int order)

{

int i;

int bad = 0;

trace_mm_page_free(page, order);

kmemcheck_free_shadow(page, order);

if (PageAnon(page))

page->mapping = NULL;

for (i = 0; i < (1 << order); i++)

bad += free_pages_check(page + i);

if (bad)

return false;

if (!PageHighMem(page)) {

debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);

debug_check_no_obj_freed(page_address(page),

PAGE_SIZE << order);

}

arch_free_page(page, order);

kernel_map_pages(page, 1 << order, 0);

return true;

}

static void __free_pages_ok(struct page *page, unsigned int order)

{

unsigned long flags;

int migratetype;

if (!free_pages_prepare(page, order))

return;

local_irq_save(flags);

__count_vm_events(PGFREE, 1 << order);

migratetype = get_pageblock_migratetype(page);

set_freepage_migratetype(page, migratetype);

free_one_page(page_zone(page), page, order, migratetype);

local_irq_restore(flags);

}

void __meminit __free_pages_bootmem(struct page *page, unsigned int order)

{

unsigned int nr_pages = 1 << order;

unsigned int loop;

prefetchw(page);

for (loop = 0; loop < nr_pages; loop++) {

struct page *p = &page[loop];

if (loop + 1 < nr_pages)

prefetchw(p + 1);

__ClearPageReserved(p);

set_page_count(p, 0);

}

set_page_refcounted(page);

__free_pages(page, order);

}

#ifdef CONFIG_CMA

/* Free whole pageblock and set it's migration type to MIGRATE_CMA. */

void __init init_cma_reserved_pageblock(struct page *page)

{

unsigned i = pageblock_nr_pages;

struct page *p = page;

do {

__ClearPageReserved(p);

set_page_count(p, 0);

} while (++p, --i);

set_page_refcounted(page);

set_pageblock_migratetype(page, MIGRATE_CMA);

__free_pages(page, pageblock_order);

totalram_pages += pageblock_nr_pages;

}

#endif

/*

* The order of subdivision here is critical for the IO subsystem.

* Please do not alter this order without good reasons and regression

* testing. Specifically, as large blocks of memory are subdivided,

* the order in which smaller blocks are delivered depends on the order

* they're subdivided in this function. This is the primary factor

* influencing the order in which pages are delivered to the IO

* subsystem according to empirical testing, and this is also justified

* by considering the behavior of a buddy system containing a single

* large block of memory acted on by a series of small allocations.

* This behavior is a critical factor in sglist merging's success.

*

* -- wli

*/

static inline void expand(struct zone *zone, struct page *page,

int low, int high, struct free_area *area,

int migratetype)

{

unsigned long size = 1 << high;

while (high > low) {

area--;

high--;

size >>= 1;

VM_BUG_ON(bad_range(zone, &page[size]));

#ifdef CONFIG_DEBUG_PAGEALLOC

if (high < debug_guardpage_minorder()) {

/*

* Mark as guard pages (or page), that will allow to

* merge back to allocator when buddy will be freed.

* Corresponding page table entries will not be touched,

* pages will stay not present in virtual address space

*/

INIT_LIST_HEAD(&page[size].lru);

set_page_guard_flag(&page[size]);

set_page_private(&page[size], high);

/* Guard pages are not available for any usage */

__mod_zone_freepage_state(zone, -(1 << high),

migratetype);

continue;

}

#endif

list_add(&page[size].lru, &area->free_list[migratetype]);

area->nr_free++;

set_page_order(&page[size], high);

}

}

/*

* This page is about to be returned from the page allocator

*/

static inline int check_new_page(struct page *page)

{

if (unlikely(page_mapcount(page) |

(page->mapping != NULL) |

(atomic_read(&page->_count) != 0) |

(page->flags & PAGE_FLAGS_CHECK_AT_PREP) |

(mem_cgroup_bad_page_check(page)))) {

bad_page(page);

return 1;

}

return 0;

}

static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)

{

int i;

for (i = 0; i < (1 << order); i++) {

struct page *p = page + i;

if (unlikely(check_new_page(p)))

return 1;

}

set_page_private(page, 0);

set_page_refcounted(page);

arch_alloc_page(page, order);

kernel_map_pages(page, 1 << order, 1);

if (gfp_flags & __GFP_ZERO)

prep_zero_page(page, order, gfp_flags);

if (order && (gfp_flags & __GFP_COMP))

prep_compound_page(page, order);

return 0;

}

/*

* Go through the free lists for the given migratetype and remove

* the smallest available page from the freelists

*/

static inline

struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,

int migratetype)

{

unsigned int current_order;

struct free_area * area;

struct page *page;

/* Find a page of the appropriate size in the preferred list */

for (current_order = order; current_order < MAX_ORDER; ++current_order) {

area = &(zone->free_area[current_order]);

if (list_empty(&area->free_list[migratetype]))

continue;

page = list_entry(area->free_list[migratetype].next,

struct page, lru);

list_del(&page->lru);

rmv_page_order(page);

area->nr_free--;

expand(zone, page, order, current_order, area, migratetype);

return page;

}

return NULL;

}

/*

* This array describes the order lists are fallen back to when

* the free lists for the desirable migrate type are depleted

*/

static int fallbacks[MIGRATE_TYPES][4] = {

[MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },

[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },

#ifdef CONFIG_CMA

[MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },

[MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */

#else

[MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },

#endif

[MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */

[MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */

};

/*

* Move the free pages in a range to the free lists of the requested type.

* Note that start_page and end_pages are not aligned on a pageblock

* boundary. If alignment is required, use move_freepages_block()

*/

int move_freepages(struct zone *zone,

struct page *start_page, struct page *end_page,

int migratetype)

{

struct page *page;

unsigned long order;

int pages_moved = 0;

#ifndef CONFIG_HOLES_IN_ZONE

/*

* page_zone is not safe to call in this context when

* CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant

* anyway as we check zone boundaries in move_freepages_block().

* Remove at a later date when no bug reports exist related to

* grouping pages by mobility

*/

BUG_ON(page_zone(start_page) != page_zone(end_page));

#endif

for (page = start_page; page <= end_page;) {

/* Make sure we are not inadvertently changing nodes */

VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));

if (!pfn_valid_within(page_to_pfn(page))) {

page++;

continue;

}

if (!PageBuddy(page)) {

page++;

continue;

}

order = page_order(page);

list_move(&page->lru,

&zone->free_area[order].free_list[migratetype]);

set_freepage_migratetype(page, migratetype);

page += 1 << order;

pages_moved += 1 << order;

}

return pages_moved;

}

int move_freepages_block(struct zone *zone, struct page *page,

int migratetype)

{

unsigned long start_pfn, end_pfn;

struct page *start_page, *end_page;

start_pfn = page_to_pfn(page);

start_pfn = start_pfn & ~(pageblock_nr_pages-1);

start_page = pfn_to_page(start_pfn);

end_page = start_page + pageblock_nr_pages - 1;

end_pfn = start_pfn + pageblock_nr_pages - 1;

/* Do not cross zone boundaries */

if (start_pfn < zone->zone_start_pfn)

start_page = page;

if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)

return 0;

return move_freepages(zone, start_page, end_page, migratetype);

}

static void change_pageblock_range(struct page *pageblock_page,

int start_order, int migratetype)

{

int nr_pageblocks = 1 << (start_order - pageblock_order);

while (nr_pageblocks--) {

set_pageblock_migratetype(pageblock_page, migratetype);

pageblock_page += pageblock_nr_pages;

}

}

/* Remove an element from the buddy allocator from the fallback list */

static inline struct page *

__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)

{

struct free_area * area;

int current_order;

struct page *page;

int migratetype, i;

/* Find the largest possible block of pages in the other list */

for (current_order = MAX_ORDER-1; current_order >= order;