// SPDX-License-Identifier: GPL-2.0-only /* * mm_init.c - Memory initialisation verification and debugging * * Copyright 2008 IBM Corporation, 2008 * Author Mel Gorman <[email protected]> * */ #include <linux/kernel.h> #include <linux/init.h> #include <linux/kobject.h> #include <linux/export.h> #include <linux/memory.h> #include <linux/notifier.h> #include <linux/sched.h> #include <linux/mman.h> #include <linux/memblock.h> #include <linux/page-isolation.h> #include <linux/padata.h> #include <linux/nmi.h> #include <linux/buffer_head.h> #include <linux/kmemleak.h> #include <linux/kfence.h> #include <linux/page_ext.h> #include <linux/pti.h> #include <linux/pgtable.h> #include <linux/stackdepot.h> #include <linux/swap.h> #include <linux/cma.h> #include <linux/crash_dump.h> #include <linux/execmem.h> #include <linux/vmstat.h> #include "internal.h" #include "slab.h" #include "shuffle.h" #include <asm/setup.h> #ifdef CONFIG_DEBUG_MEMORY_INIT int __meminitdata mminit_loglevel; /* The zonelists are simply reported, validation is manual. */ void __init mminit_verify_zonelist(void) { … } void __init mminit_verify_pageflags_layout(void) { … } static __init int set_mminit_loglevel(char *str) { … } early_param(…); #endif /* CONFIG_DEBUG_MEMORY_INIT */ struct kobject *mm_kobj; #ifdef CONFIG_SMP s32 vm_committed_as_batch = …; void mm_compute_batch(int overcommit_policy) { … } static int __meminit mm_compute_batch_notifier(struct notifier_block *self, unsigned long action, void *arg) { … } static int __init mm_compute_batch_init(void) { … } __initcall(mm_compute_batch_init); #endif static int __init mm_sysfs_init(void) { … } postcore_initcall(mm_sysfs_init); static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; static unsigned long required_kernelcore __initdata; static unsigned long required_kernelcore_percent __initdata; static unsigned long required_movablecore __initdata; static unsigned long required_movablecore_percent __initdata; static unsigned long nr_kernel_pages __initdata; static unsigned long nr_all_pages __initdata; static bool deferred_struct_pages __meminitdata; static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); static int __init cmdline_parse_core(char *p, unsigned long *core, unsigned long *percent) { … } bool mirrored_kernelcore __initdata_memblock; /* * kernelcore=size sets the amount of memory for use for allocations that * cannot be reclaimed or migrated. */ static int __init cmdline_parse_kernelcore(char *p) { … } early_param(…); /* * movablecore=size sets the amount of memory for use for allocations that * can be reclaimed or migrated. */ static int __init cmdline_parse_movablecore(char *p) { … } early_param(…); /* * early_calculate_totalpages() * Sum pages in active regions for movable zone. * Populate N_MEMORY for calculating usable_nodes. */ static unsigned long __init early_calculate_totalpages(void) { … } /* * This finds a zone that can be used for ZONE_MOVABLE pages. The * assumption is made that zones within a node are ordered in monotonic * increasing memory addresses so that the "highest" populated zone is used */ static void __init find_usable_zone_for_movable(void) { … } /* * Find the PFN the Movable zone begins in each node. Kernel memory * is spread evenly between nodes as long as the nodes have enough * memory. When they don't, some nodes will have more kernelcore than * others */ static void __init find_zone_movable_pfns_for_nodes(void) { … } void __meminit __init_single_page(struct page *page, unsigned long pfn, unsigned long zone, int nid) { … } #ifdef CONFIG_NUMA /* * During memory init memblocks map pfns to nids. The search is expensive and * this caches recent lookups. The implementation of __early_pfn_to_nid * treats start/end as pfns. */ struct mminit_pfnnid_cache { … }; static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; /* * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. */ static int __meminit __early_pfn_to_nid(unsigned long pfn, struct mminit_pfnnid_cache *state) { … } int __meminit early_pfn_to_nid(unsigned long pfn) { … } int hashdist = … HASHDIST_DEFAULT; static int __init set_hashdist(char *str) { … } __setup(…); static inline void fixup_hashdist(void) { … } #else static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { … } /* Returns true if the struct page for the pfn is initialised */ static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) { … } /* * Returns true when the remaining initialisation should be deferred until * later in the boot cycle when it can be parallelised. */ static bool __meminit defer_init(int nid, unsigned long pfn, unsigned long end_pfn) { … } static void __meminit init_reserved_page(unsigned long pfn, int nid) { … } #else static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} static inline bool early_page_initialised(unsigned long pfn, int nid) { return true; } static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) { return false; } static inline void init_reserved_page(unsigned long pfn, int nid) { } #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ /* * Initialised pages do not have PageReserved set. This function is * called for each range allocated by the bootmem allocator and * marks the pages PageReserved. The remaining valid pages are later * sent to the buddy page allocator. */ void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int nid) { … } /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ static bool __meminit overlap_memmap_init(unsigned long zone, unsigned long *pfn) { … } /* * Only struct pages that correspond to ranges defined by memblock.memory * are zeroed and initialized by going through __init_single_page() during * memmap_init_zone_range(). * * But, there could be struct pages that correspond to holes in * memblock.memory. This can happen because of the following reasons: * - physical memory bank size is not necessarily the exact multiple of the * arbitrary section size * - early reserved memory may not be listed in memblock.memory * - non-memory regions covered by the contigious flatmem mapping * - memory layouts defined with memmap= kernel parameter may not align * nicely with memmap sections * * Explicitly initialize those struct pages so that: * - PG_Reserved is set * - zone and node links point to zone and node that span the page if the * hole is in the middle of a zone * - zone and node links point to adjacent zone/node if the hole falls on * the zone boundary; the pages in such holes will be prepended to the * zone/node above the hole except for the trailing pages in the last * section that will be appended to the zone/node below. */ static void __init init_unavailable_range(unsigned long spfn, unsigned long epfn, int zone, int node) { … } /* * Initially all pages are reserved - free ones are freed * up by memblock_free_all() once the early boot process is * done. Non-atomic initialization, single-pass. * * All aligned pageblocks are initialized to the specified migratetype * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related * zone stats (e.g., nr_isolate_pageblock) are touched. */ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, unsigned long start_pfn, unsigned long zone_end_pfn, enum meminit_context context, struct vmem_altmap *altmap, int migratetype) { … } static void __init memmap_init_zone_range(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn, unsigned long *hole_pfn) { … } static void __init memmap_init(void) { … } #ifdef CONFIG_ZONE_DEVICE static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap) { … } /* * With compound page geometry and when struct pages are stored in ram most * tail pages are reused. Consequently, the amount of unique struct pages to * initialize is a lot smaller that the total amount of struct pages being * mapped. This is a paired / mild layering violation with explicit knowledge * of how the sparse_vmemmap internals handle compound pages in the lack * of an altmap. See vmemmap_populate_compound_pages(). */ static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { … } static void __ref memmap_init_compound(struct page *head, unsigned long head_pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap, unsigned long nr_pages) { … } void __ref memmap_init_zone_device(struct zone *zone, unsigned long start_pfn, unsigned long nr_pages, struct dev_pagemap *pgmap) { … } #endif /* * The zone ranges provided by the architecture do not include ZONE_MOVABLE * because it is sized independent of architecture. Unlike the other zones, * the starting point for ZONE_MOVABLE is not fixed. It may be different * in each node depending on the size of each node and how evenly kernelcore * is distributed. This helper function adjusts the zone ranges * provided by the architecture for a given node by using the end of the * highest usable zone for ZONE_MOVABLE. This preserves the assumption that * zones within a node are in order of monotonic increases memory addresses */ static void __init adjust_zone_range_for_zone_movable(int nid, unsigned long zone_type, unsigned long node_end_pfn, unsigned long *zone_start_pfn, unsigned long *zone_end_pfn) { … } /* * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, * then all holes in the requested range will be accounted for. */ static unsigned long __init __absent_pages_in_range(int nid, unsigned long range_start_pfn, unsigned long range_end_pfn) { … } /** * absent_pages_in_range - Return number of page frames in holes within a range * @start_pfn: The start PFN to start searching for holes * @end_pfn: The end PFN to stop searching for holes * * Return: the number of pages frames in memory holes within a range. */ unsigned long __init absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn) { … } /* Return the number of page frames in holes in a zone on a node */ static unsigned long __init zone_absent_pages_in_node(int nid, unsigned long zone_type, unsigned long zone_start_pfn, unsigned long zone_end_pfn) { … } /* * Return the number of pages a zone spans in a node, including holes * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() */ static unsigned long __init zone_spanned_pages_in_node(int nid, unsigned long zone_type, unsigned long node_start_pfn, unsigned long node_end_pfn, unsigned long *zone_start_pfn, unsigned long *zone_end_pfn) { … } static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) { … } static void __init calc_nr_kernel_pages(void) { … } static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) { … } #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void pgdat_init_split_queue(struct pglist_data *pgdat) { … } #else static void pgdat_init_split_queue(struct pglist_data *pgdat) {} #endif #ifdef CONFIG_COMPACTION static void pgdat_init_kcompactd(struct pglist_data *pgdat) { … } #else static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} #endif static void __meminit pgdat_init_internals(struct pglist_data *pgdat) { … } static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, unsigned long remaining_pages) { … } static void __meminit zone_init_free_lists(struct zone *zone) { … } void __meminit init_currently_empty_zone(struct zone *zone, unsigned long zone_start_pfn, unsigned long size) { … } #ifndef CONFIG_SPARSEMEM /* * Calculate the size of the zone->blockflags rounded to an unsigned long * Start by making sure zonesize is a multiple of pageblock_order by rounding * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally * round what is now in bits to nearest long in bits, then return it in * bytes. */ static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) { unsigned long usemapsize; zonesize += zone_start_pfn & (pageblock_nr_pages-1); usemapsize = roundup(zonesize, pageblock_nr_pages); usemapsize = usemapsize >> pageblock_order; usemapsize *= NR_PAGEBLOCK_BITS; usemapsize = roundup(usemapsize, BITS_PER_LONG); return usemapsize / BITS_PER_BYTE; } static void __ref setup_usemap(struct zone *zone) { unsigned long usemapsize = usemap_size(zone->zone_start_pfn, zone->spanned_pages); zone->pageblock_flags = NULL; if (usemapsize) { zone->pageblock_flags = memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, zone_to_nid(zone)); if (!zone->pageblock_flags) panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", usemapsize, zone->name, zone_to_nid(zone)); } } #else static inline void setup_usemap(struct zone *zone) { … } #endif /* CONFIG_SPARSEMEM */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ void __init set_pageblock_order(void) { unsigned int order = MAX_PAGE_ORDER; /* Check that pageblock_nr_pages has not already been setup */ if (pageblock_order) return; /* Don't let pageblocks exceed the maximum allocation granularity. */ if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) order = HUGETLB_PAGE_ORDER; /* * Assume the largest contiguous order of interest is a huge page. * This value may be variable depending on boot parameters on powerpc. */ pageblock_order = order; } #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ /* * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() * is unused as pageblock_order is set at compile-time. See * include/linux/pageblock-flags.h for the values of pageblock_order based on * the kernel config */ void __init set_pageblock_order(void) { … } #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ /* * Set up the zone data structures * - init pgdat internals * - init all zones belonging to this node * * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { … } #endif static void __init free_area_init_core(struct pglist_data *pgdat) { … } void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, int nid, bool exact_nid) { … } #ifdef CONFIG_FLATMEM static void __init alloc_node_mem_map(struct pglist_data *pgdat) { unsigned long start, offset, size, end; struct page *map; /* Skip empty nodes */ if (!pgdat->node_spanned_pages) return; start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); offset = pgdat->node_start_pfn - start; /* * The zone's endpoints aren't required to be MAX_PAGE_ORDER * aligned but the node_mem_map endpoints must be in order * for the buddy allocator to function correctly. */ end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES); size = (end - start) * sizeof(struct page); map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, pgdat->node_id, false); if (!map) panic("Failed to allocate %ld bytes for node %d memory map\n", size, pgdat->node_id); pgdat->node_mem_map = map + offset; mod_node_early_perpage_metadata(pgdat->node_id, DIV_ROUND_UP(size, PAGE_SIZE)); pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", __func__, pgdat->node_id, (unsigned long)pgdat, (unsigned long)pgdat->node_mem_map); #ifndef CONFIG_NUMA /* the global mem_map is just set as node 0's */ if (pgdat == NODE_DATA(0)) { mem_map = NODE_DATA(0)->node_mem_map; if (page_to_pfn(mem_map) != pgdat->node_start_pfn) mem_map -= offset; } #endif } #else static inline void alloc_node_mem_map(struct pglist_data *pgdat) { … } #endif /* CONFIG_FLATMEM */ /** * get_pfn_range_for_nid - Return the start and end page frames for a node * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. * @start_pfn: Passed by reference. On return, it will have the node start_pfn. * @end_pfn: Passed by reference. On return, it will have the node end_pfn. * * It returns the start and end page frame of a node based on information * provided by memblock_set_node(). If called for a node * with no available memory, the start and end PFNs will be 0. */ void __init get_pfn_range_for_nid(unsigned int nid, unsigned long *start_pfn, unsigned long *end_pfn) { … } static void __init free_area_init_node(int nid) { … } /* Any regular or high memory on that node ? */ static void __init check_for_memory(pg_data_t *pgdat) { … } #if MAX_NUMNODES > 1 /* * Figure out the number of possible node ids. */ void __init setup_nr_node_ids(void) { … } #endif /* * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For * such cases we allow max_zone_pfn sorted in the descending order */ static bool arch_has_descending_max_zone_pfns(void) { … } /** * free_area_init - Initialise all pg_data_t and zone data * @max_zone_pfn: an array of max PFNs for each zone * * This will call free_area_init_node() for each active node in the system. * Using the page ranges provided by memblock_set_node(), the size of each * zone in each node and their holes is calculated. If the maximum PFN * between two adjacent zones match, it is assumed that the zone is empty. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed * that arch_max_dma32_pfn has no pages. It is also assumed that a zone * starts where the previous one ended. For example, ZONE_DMA32 starts * at arch_max_dma_pfn. */ void __init free_area_init(unsigned long *max_zone_pfn) { … } /** * node_map_pfn_alignment - determine the maximum internode alignment * * This function should be called after node map is populated and sorted. * It calculates the maximum power of two alignment which can distinguish * all the nodes. * * For example, if all nodes are 1GiB and aligned to 1GiB, the return value * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is * shifted, 1GiB is enough and this function will indicate so. * * This is used to test whether pfn -> nid mapping of the chosen memory * model has fine enough granularity to avoid incorrect mapping for the * populated node map. * * Return: the determined alignment in pfn's. 0 if there is no alignment * requirement (single node). */ unsigned long __init node_map_pfn_alignment(void) { … } #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static void __init deferred_free_pages(unsigned long pfn, unsigned long nr_pages) { … } /* Completion tracking for deferred_init_memmap() threads */ static atomic_t pgdat_init_n_undone __initdata; static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); static inline void __init pgdat_init_report_one_done(void) { … } /* * Initialize struct pages. We minimize pfn page lookups and scheduler checks * by performing it only once every MAX_ORDER_NR_PAGES. * Return number of pages initialized. */ static unsigned long __init deferred_init_pages(struct zone *zone, unsigned long pfn, unsigned long end_pfn) { … } /* * This function is meant to pre-load the iterator for the zone init from * a given point. * Specifically it walks through the ranges starting with initial index * passed to it until we are caught up to the first_init_pfn value and * exits there. If we never encounter the value we return false indicating * there are no valid ranges left. */ static bool __init deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, unsigned long *spfn, unsigned long *epfn, unsigned long first_init_pfn) { … } /* * Initialize and free pages. We do it in two loops: first we initialize * struct page, then free to buddy allocator, because while we are * freeing pages we can access pages that are ahead (computing buddy * page in __free_one_page()). * * In order to try and keep some memory in the cache we have the loop * broken along max page order boundaries. This way we will not cause * any issues with the buddy page computation. */ static unsigned long __init deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, unsigned long *end_pfn) { … } static void __init deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, void *arg) { … } static unsigned int __init deferred_page_init_max_threads(const struct cpumask *node_cpumask) { … } /* Initialise remaining memory on a node */ static int __init deferred_init_memmap(void *data) { … } /* * If this zone has deferred pages, try to grow it by initializing enough * deferred pages to satisfy the allocation specified by order, rounded up to * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments * of SECTION_SIZE bytes by initializing struct pages in increments of * PAGES_PER_SECTION * sizeof(struct page) bytes. * * Return true when zone was grown, otherwise return false. We return true even * when we grow less than requested, to let the caller decide if there are * enough pages to satisfy the allocation. */ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) { … } #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ #ifdef CONFIG_CMA void __init init_cma_reserved_pageblock(struct page *page) { … } #endif void set_zone_contiguous(struct zone *zone) { … } static void __init mem_init_print_info(void); void __init page_alloc_init_late(void) { … } /* * Adaptive scale is meant to reduce sizes of hash tables on large memory * machines. As memory size is increased the scale is also increased but at * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory * quadruples the scale is increased by one, which means the size of hash table * only doubles, instead of quadrupling as well. * Because 32-bit systems cannot have large physical memory, where this scaling * makes sense, it is disabled on such platforms. */ #if __BITS_PER_LONG > 32 #define ADAPT_SCALE_BASE … #define ADAPT_SCALE_SHIFT … #define ADAPT_SCALE_NPAGES … #endif /* * allocate a large system hash table from bootmem * - it is assumed that the hash table must contain an exact power-of-2 * quantity of entries * - limit is the number of hash buckets, not the total allocation size */ void *__init alloc_large_system_hash(const char *tablename, unsigned long bucketsize, unsigned long numentries, int scale, int flags, unsigned int *_hash_shift, unsigned int *_hash_mask, unsigned long low_limit, unsigned long high_limit) { … } void __init memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order) { … } DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); EXPORT_SYMBOL(…); DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); EXPORT_SYMBOL(…); static bool _init_on_alloc_enabled_early __read_mostly = … IS_ENABLED(…); static int __init early_init_on_alloc(char *buf) { … } early_param(…); static bool _init_on_free_enabled_early __read_mostly = … IS_ENABLED(…); static int __init early_init_on_free(char *buf) { … } early_param(…); DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); /* * Enable static keys related to various memory debugging and hardening options. * Some override others, and depend on early params that are evaluated in the * order of appearance. So we need to first gather the full picture of what was * enabled, and then make decisions. */ static void __init mem_debugging_and_hardening_init(void) { … } /* Report memory auto-initialization states for this boot. */ static void __init report_meminit(void) { … } static void __init mem_init_print_info(void) { … } /* * Set up kernel memory allocators */ void __init mm_core_init(void) { … }
Codebase Browser
linux