// SPDX-License-Identifier: GPL-2.0 /* * linux/mm/page_isolation.c */ #include <linux/mm.h> #include <linux/page-isolation.h> #include <linux/pageblock-flags.h> #include <linux/memory.h> #include <linux/hugetlb.h> #include <linux/page_owner.h> #include <linux/migrate.h> #include "internal.h" #define CREATE_TRACE_POINTS #include <trace/events/page_isolation.h> /* * This function checks whether the range [start_pfn, end_pfn) includes * unmovable pages or not. The range must fall into a single pageblock and * consequently belong to a single zone. * * PageLRU check without isolation or lru_lock could race so that * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable * check without lock_page also may miss some movable non-lru pages at * race condition. So you can't expect this function should be exact. * * Returns a page without holding a reference. If the caller wants to * dereference that page (e.g., dumping), it has to make sure that it * cannot get removed (e.g., via memory unplug) concurrently. * */ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long end_pfn, int migratetype, int flags) { … } /* * This function set pageblock migratetype to isolate if no unmovable page is * present in [start_pfn, end_pfn). The pageblock must intersect with * [start_pfn, end_pfn). */ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_flags, unsigned long start_pfn, unsigned long end_pfn) { … } static void unset_migratetype_isolate(struct page *page, int migratetype) { … } static inline struct page * __first_valid_page(unsigned long pfn, unsigned long nr_pages) { … } /** * isolate_single_pageblock() -- tries to isolate a pageblock that might be * within a free or in-use page. * @boundary_pfn: pageblock-aligned pfn that a page might cross * @flags: isolation flags * @gfp_flags: GFP flags used for migrating pages * @isolate_before: isolate the pageblock before the boundary_pfn * @skip_isolation: the flag to skip the pageblock isolation in second * isolate_single_pageblock() * @migratetype: migrate type to set in error recovery. * * Free and in-use pages can be as big as MAX_PAGE_ORDER and contain more than one * pageblock. When not all pageblocks within a page are isolated at the same * time, free page accounting can go wrong. For example, in the case of * MAX_PAGE_ORDER = pageblock_order + 1, a MAX_PAGE_ORDER page has two * pagelbocks. * [ MAX_PAGE_ORDER ] * [ pageblock0 | pageblock1 ] * When either pageblock is isolated, if it is a free page, the page is not * split into separate migratetype lists, which is supposed to; if it is an * in-use page and freed later, __free_one_page() does not split the free page * either. The function handles this by splitting the free page or migrating * the in-use page then splitting the free page. */ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, gfp_t gfp_flags, bool isolate_before, bool skip_isolation, int migratetype) { … } /** * start_isolate_page_range() - mark page range MIGRATE_ISOLATE * @start_pfn: The first PFN of the range to be isolated. * @end_pfn: The last PFN of the range to be isolated. * @migratetype: Migrate type to set in error recovery. * @flags: The following flags are allowed (they can be combined in * a bit mask) * MEMORY_OFFLINE - isolate to offline (!allocate) memory * e.g., skip over PageHWPoison() pages * and PageOffline() pages. * REPORT_FAILURE - report details about the failure to * isolate the range * @gfp_flags: GFP flags used for migrating pages that sit across the * range boundaries. * * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in * the range will never be allocated. Any free pages and pages freed in the * future will not be allocated again. If specified range includes migrate types * other than MOVABLE or CMA, this will fail with -EBUSY. For isolating all * pages in the range finally, the caller have to free all pages in the range. * test_page_isolated() can be used for test it. * * The function first tries to isolate the pageblocks at the beginning and end * of the range, since there might be pages across the range boundaries. * Afterwards, it isolates the rest of the range. * * There is no high level synchronization mechanism that prevents two threads * from trying to isolate overlapping ranges. If this happens, one thread * will notice pageblocks in the overlapping range already set to isolate. * This happens in set_migratetype_isolate, and set_migratetype_isolate * returns an error. We then clean up by restoring the migration type on * pageblocks we may have modified and return -EBUSY to caller. This * prevents two threads from simultaneously working on overlapping ranges. * * Please note that there is no strong synchronization with the page allocator * either. Pages might be freed while their page blocks are marked ISOLATED. * A call to drain_all_pages() after isolation can flush most of them. However * in some cases pages might still end up on pcp lists and that would allow * for their allocation even when they are in fact isolated already. Depending * on how strong of a guarantee the caller needs, zone_pcp_disable/enable() * might be used to flush and disable pcplist before isolation and enable after * unisolation. * * Return: 0 on success and -EBUSY if any part of range cannot be isolated. */ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, int migratetype, int flags, gfp_t gfp_flags) { … } /** * undo_isolate_page_range - undo effects of start_isolate_page_range() * @start_pfn: The first PFN of the isolated range * @end_pfn: The last PFN of the isolated range * @migratetype: New migrate type to set on the range * * This finds every MIGRATE_ISOLATE page block in the given range * and switches it to @migratetype. */ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, int migratetype) { … } /* * Test all pages in the range is free(means isolated) or not. * all pages in [start_pfn...end_pfn) must be in the same zone. * zone->lock must be held before call this. * * Returns the last tested pfn. */ static unsigned long __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, int flags) { … } /** * test_pages_isolated - check if pageblocks in range are isolated * @start_pfn: The first PFN of the isolated range * @end_pfn: The first PFN *after* the isolated range * @isol_flags: Testing mode flags * * This tests if all in the specified range are free. * * If %MEMORY_OFFLINE is specified in @flags, it will consider * poisoned and offlined pages free as well. * * Caller must ensure the requested range doesn't span zones. * * Returns 0 if true, -EBUSY if one or more pages are in use. */ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, int isol_flags) { … }
Codebase Browser
linux