// SPDX-License-Identifier: GPL-2.0 /* * Memory subsystem support * * Written by Matt Tolentino <[email protected]> * Dave Hansen <[email protected]> * * This file provides the necessary infrastructure to represent * a SPARSEMEM-memory-model system's physical memory in /sysfs. * All arch-independent code that assumes MEMORY_HOTPLUG requires * SPARSEMEM should be contained here, or in mm/memory_hotplug.c. */ #include <linux/module.h> #include <linux/init.h> #include <linux/topology.h> #include <linux/capability.h> #include <linux/device.h> #include <linux/memory.h> #include <linux/memory_hotplug.h> #include <linux/mm.h> #include <linux/stat.h> #include <linux/slab.h> #include <linux/xarray.h> #include <linux/atomic.h> #include <linux/uaccess.h> #define MEMORY_CLASS_NAME … static const char *const online_type_to_str[] = …; int mhp_online_type_from_str(const char *str) { … } #define to_memory_block(dev) … static int sections_per_block; static inline unsigned long memory_block_id(unsigned long section_nr) { … } static inline unsigned long pfn_to_block_id(unsigned long pfn) { … } static inline unsigned long phys_to_block_id(unsigned long phys) { … } static int memory_subsys_online(struct device *dev); static int memory_subsys_offline(struct device *dev); static const struct bus_type memory_subsys = …; /* * Memory blocks are cached in a local radix tree to avoid * a costly linear search for the corresponding device on * the subsystem bus. */ static DEFINE_XARRAY(memory_blocks); /* * Memory groups, indexed by memory group id (mgid). */ static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC); #define MEMORY_GROUP_MARK_DYNAMIC … static BLOCKING_NOTIFIER_HEAD(memory_chain); int register_memory_notifier(struct notifier_block *nb) { … } EXPORT_SYMBOL(…); void unregister_memory_notifier(struct notifier_block *nb) { … } EXPORT_SYMBOL(…); static void memory_block_release(struct device *dev) { … } unsigned long __weak memory_block_size_bytes(void) { … } EXPORT_SYMBOL_GPL(…); /* Show the memory block ID, relative to the memory block size */ static ssize_t phys_index_show(struct device *dev, struct device_attribute *attr, char *buf) { … } /* * Legacy interface that we cannot remove. Always indicate "removable" * with CONFIG_MEMORY_HOTREMOVE - bad heuristic. */ static ssize_t removable_show(struct device *dev, struct device_attribute *attr, char *buf) { … } /* * online, offline, going offline, etc. */ static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf) { … } int memory_notify(unsigned long val, void *v) { … } #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) static unsigned long memblk_nr_poison(struct memory_block *mem); #else static inline unsigned long memblk_nr_poison(struct memory_block *mem) { return 0; } #endif /* * Must acquire mem_hotplug_lock in write mode. */ static int memory_block_online(struct memory_block *mem) { … } /* * Must acquire mem_hotplug_lock in write mode. */ static int memory_block_offline(struct memory_block *mem) { … } /* * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is * OK to have direct references to sparsemem variables in here. */ static int memory_block_action(struct memory_block *mem, unsigned long action) { … } static int memory_block_change_state(struct memory_block *mem, unsigned long to_state, unsigned long from_state_req) { … } /* The device lock serializes operations on memory_subsys_[online|offline] */ static int memory_subsys_online(struct device *dev) { … } static int memory_subsys_offline(struct device *dev) { … } static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } /* * Legacy interface that we cannot remove: s390x exposes the storage increment * covered by a memory block, allowing for identifying which memory blocks * comprise a storage increment. Since a memory block spans complete * storage increments nowadays, this interface is basically unused. Other * archs never exposed != 0. */ static ssize_t phys_device_show(struct device *dev, struct device_attribute *attr, char *buf) { … } #ifdef CONFIG_MEMORY_HOTREMOVE static int print_allowed_zone(char *buf, int len, int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages, int online_type, struct zone *default_zone) { … } static ssize_t valid_zones_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(valid_zones); #endif static DEVICE_ATTR_RO(phys_index); static DEVICE_ATTR_RW(state); static DEVICE_ATTR_RO(phys_device); static DEVICE_ATTR_RO(removable); /* * Show the memory block size (shared by all memory blocks). */ static ssize_t block_size_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(block_size_bytes); /* * Memory auto online policy. */ static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } static DEVICE_ATTR_RW(auto_online_blocks); #ifdef CONFIG_CRASH_HOTPLUG #include <linux/kexec.h> static ssize_t crash_hotplug_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(crash_hotplug); #endif /* * Some architectures will have custom drivers to do this, and * will not need to do it from userspace. The fake hot-add code * as well as ppc64 will do all of their discovery in userspace * and will require this interface. */ #ifdef CONFIG_ARCH_MEMORY_PROBE static ssize_t probe_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } static DEVICE_ATTR_WO(probe); #endif #ifdef CONFIG_MEMORY_FAILURE /* * Support for offlining pages of memory */ /* Soft offline a page */ static ssize_t soft_offline_page_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } /* Forcibly offline a page, including killing processes. */ static ssize_t hard_offline_page_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { … } static DEVICE_ATTR_WO(soft_offline_page); static DEVICE_ATTR_WO(hard_offline_page); #endif /* See phys_device_show(). */ int __weak arch_get_memory_phys_device(unsigned long start_pfn) { … } /* * A reference for the returned memory block device is acquired. * * Called under device_hotplug_lock. */ static struct memory_block *find_memory_block_by_id(unsigned long block_id) { … } /* * Called under device_hotplug_lock. */ struct memory_block *find_memory_block(unsigned long section_nr) { … } static struct attribute *memory_memblk_attrs[] = …; static const struct attribute_group memory_memblk_attr_group = …; static const struct attribute_group *memory_memblk_attr_groups[] = …; static int __add_memory_block(struct memory_block *memory) { … } static struct zone *early_node_zone_for_memory_block(struct memory_block *mem, int nid) { … } #ifdef CONFIG_NUMA /** * memory_block_add_nid() - Indicate that system RAM falling into this memory * block device (partially) belongs to the given node. * @mem: The memory block device. * @nid: The node id. * @context: The memory initialization context. * * Indicate that system RAM falling into this memory block (partially) belongs * to the given node. If the context indicates ("early") that we are adding the * node during node device subsystem initialization, this will also properly * set/adjust mem->zone based on the zone ranges of the given node. */ void memory_block_add_nid(struct memory_block *mem, int nid, enum meminit_context context) { … } #endif static int add_memory_block(unsigned long block_id, unsigned long state, struct vmem_altmap *altmap, struct memory_group *group) { … } static int __init add_boot_memory_block(unsigned long base_section_nr) { … } static int add_hotplug_memory_block(unsigned long block_id, struct vmem_altmap *altmap, struct memory_group *group) { … } static void remove_memory_block(struct memory_block *memory) { … } /* * Create memory block devices for the given memory area. Start and size * have to be aligned to memory block granularity. Memory block devices * will be initialized as offline. * * Called under device_hotplug_lock. */ int create_memory_block_devices(unsigned long start, unsigned long size, struct vmem_altmap *altmap, struct memory_group *group) { … } /* * Remove memory block devices for the given memory area. Start and size * have to be aligned to memory block granularity. Memory block devices * have to be offline. * * Called under device_hotplug_lock. */ void remove_memory_block_devices(unsigned long start, unsigned long size) { … } static struct attribute *memory_root_attrs[] = …; static const struct attribute_group memory_root_attr_group = …; static const struct attribute_group *memory_root_attr_groups[] = …; /* * Initialize the sysfs support for memory devices. At the time this function * is called, we cannot have concurrent creation/deletion of memory block * devices, the device_hotplug_lock is not needed. */ void __init memory_dev_init(void) { … } /** * walk_memory_blocks - walk through all present memory blocks overlapped * by the range [start, start + size) * * @start: start address of the memory range * @size: size of the memory range * @arg: argument passed to func * @func: callback for each memory section walked * * This function walks through all present memory blocks overlapped by the * range [start, start + size), calling func on each memory block. * * In case func() returns an error, walking is aborted and the error is * returned. * * Called under device_hotplug_lock. */ int walk_memory_blocks(unsigned long start, unsigned long size, void *arg, walk_memory_blocks_func_t func) { … } struct for_each_memory_block_cb_data { … }; static int for_each_memory_block_cb(struct device *dev, void *data) { … } /** * for_each_memory_block - walk through all present memory blocks * * @arg: argument passed to func * @func: callback for each memory block walked * * This function walks through all present memory blocks, calling func on * each memory block. * * In case func() returns an error, walking is aborted and the error is * returned. */ int for_each_memory_block(void *arg, walk_memory_blocks_func_t func) { … } /* * This is an internal helper to unify allocation and initialization of * memory groups. Note that the passed memory group will be copied to a * dynamically allocated memory group. After this call, the passed * memory group should no longer be used. */ static int memory_group_register(struct memory_group group) { … } /** * memory_group_register_static() - Register a static memory group. * @nid: The node id. * @max_pages: The maximum number of pages we'll have in this static memory * group. * * Register a new static memory group and return the memory group id. * All memory in the group belongs to a single unit, such as a DIMM. All * memory belonging to a static memory group is added in one go to be removed * in one go -- it's static. * * Returns an error if out of memory, if the node id is invalid, if no new * memory groups can be registered, or if max_pages is invalid (0). Otherwise, * returns the new memory group id. */ int memory_group_register_static(int nid, unsigned long max_pages) { … } EXPORT_SYMBOL_GPL(…); /** * memory_group_register_dynamic() - Register a dynamic memory group. * @nid: The node id. * @unit_pages: Unit in pages in which is memory added/removed in this dynamic * memory group. * * Register a new dynamic memory group and return the memory group id. * Memory within a dynamic memory group is added/removed dynamically * in unit_pages. * * Returns an error if out of memory, if the node id is invalid, if no new * memory groups can be registered, or if unit_pages is invalid (0, not a * power of two, smaller than a single memory block). Otherwise, returns the * new memory group id. */ int memory_group_register_dynamic(int nid, unsigned long unit_pages) { … } EXPORT_SYMBOL_GPL(…); /** * memory_group_unregister() - Unregister a memory group. * @mgid: the memory group id * * Unregister a memory group. If any memory block still belongs to this * memory group, unregistering will fail. * * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some * memory blocks still belong to this memory group and returns 0 if * unregistering succeeded. */ int memory_group_unregister(int mgid) { … } EXPORT_SYMBOL_GPL(…); /* * This is an internal helper only to be used in core memory hotplug code to * lookup a memory group. We don't care about locking, as we don't expect a * memory group to get unregistered while adding memory to it -- because * the group and the memory is managed by the same driver. */ struct memory_group *memory_group_find_by_id(int mgid) { … } /* * This is an internal helper only to be used in core memory hotplug code to * walk all dynamic memory groups excluding a given memory group, either * belonging to a specific node, or belonging to any node. */ int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, struct memory_group *excluded, void *arg) { … } #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) void memblk_nr_poison_inc(unsigned long pfn) { … } void memblk_nr_poison_sub(unsigned long pfn, long i) { … } static unsigned long memblk_nr_poison(struct memory_block *mem) { … } #endif
Codebase Browser
linux