// SPDX-License-Identifier: GPL-2.0-or-later /* * Virtio-mem device driver. * * Copyright Red Hat, Inc. 2020 * * Author(s): David Hildenbrand <[email protected]> */ #include <linux/virtio.h> #include <linux/virtio_mem.h> #include <linux/workqueue.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/mm.h> #include <linux/memory_hotplug.h> #include <linux/memory.h> #include <linux/hrtimer.h> #include <linux/crash_dump.h> #include <linux/mutex.h> #include <linux/bitmap.h> #include <linux/lockdep.h> #include <linux/log2.h> #include <linux/vmalloc.h> #include <linux/suspend.h> #include <acpi/acpi_numa.h> static bool unplug_online = …; module_param(unplug_online, bool, 0644); MODULE_PARM_DESC(…) …; static bool force_bbm; module_param(force_bbm, bool, 0444); MODULE_PARM_DESC(…) …; static unsigned long bbm_block_size; module_param(bbm_block_size, ulong, 0444); MODULE_PARM_DESC(…) …; /* * virtio-mem currently supports the following modes of operation: * * * Sub Block Mode (SBM): A Linux memory block spans 2..X subblocks (SB). The * size of a Sub Block (SB) is determined based on the device block size, the * pageblock size, and the maximum allocation granularity of the buddy. * Subblocks within a Linux memory block might either be plugged or unplugged. * Memory is added/removed to Linux MM in Linux memory block granularity. * * * Big Block Mode (BBM): A Big Block (BB) spans 1..X Linux memory blocks. * Memory is added/removed to Linux MM in Big Block granularity. * * The mode is determined automatically based on the Linux memory block size * and the device block size. * * User space / core MM (auto onlining) is responsible for onlining added * Linux memory blocks - and for selecting a zone. Linux Memory Blocks are * always onlined separately, and all memory within a Linux memory block is * onlined to the same zone - virtio-mem relies on this behavior. */ /* * State of a Linux memory block in SBM. */ enum virtio_mem_sbm_mb_state { … }; /* * State of a Big Block (BB) in BBM, covering 1..X Linux memory blocks. */ enum virtio_mem_bbm_bb_state { … }; struct virtio_mem { … }; /* * We have to share a single online_page callback among all virtio-mem * devices. We use RCU to iterate the list in the callback. */ static DEFINE_MUTEX(virtio_mem_mutex); static LIST_HEAD(virtio_mem_devices); static void virtio_mem_online_page_cb(struct page *page, unsigned int order); static void virtio_mem_fake_offline_going_offline(unsigned long pfn, unsigned long nr_pages); static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn, unsigned long nr_pages); static void virtio_mem_retry(struct virtio_mem *vm); static int virtio_mem_create_resource(struct virtio_mem *vm); static void virtio_mem_delete_resource(struct virtio_mem *vm); /* * Register a virtio-mem device so it will be considered for the online_page * callback. */ static int register_virtio_mem_device(struct virtio_mem *vm) { … } /* * Unregister a virtio-mem device so it will no longer be considered for the * online_page callback. */ static void unregister_virtio_mem_device(struct virtio_mem *vm) { … } /* * Calculate the memory block id of a given address. */ static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr) { … } /* * Calculate the physical start address of a given memory block id. */ static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id) { … } /* * Calculate the big block id of a given address. */ static unsigned long virtio_mem_phys_to_bb_id(struct virtio_mem *vm, uint64_t addr) { … } /* * Calculate the physical start address of a given big block id. */ static uint64_t virtio_mem_bb_id_to_phys(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Calculate the subblock id of a given address. */ static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm, unsigned long addr) { … } /* * Set the state of a big block, taking care of the state counter. */ static void virtio_mem_bbm_set_bb_state(struct virtio_mem *vm, unsigned long bb_id, enum virtio_mem_bbm_bb_state state) { … } /* * Get the state of a big block. */ static enum virtio_mem_bbm_bb_state virtio_mem_bbm_get_bb_state(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Prepare the big block state array for the next big block. */ static int virtio_mem_bbm_bb_states_prepare_next_bb(struct virtio_mem *vm) { … } #define virtio_mem_bbm_for_each_bb(_vm, _bb_id, _state) … #define virtio_mem_bbm_for_each_bb_rev(_vm, _bb_id, _state) … /* * Set the state of a memory block, taking care of the state counter. */ static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm, unsigned long mb_id, uint8_t state) { … } /* * Get the state of a memory block. */ static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm, unsigned long mb_id) { … } /* * Prepare the state array for the next memory block. */ static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm) { … } #define virtio_mem_sbm_for_each_mb(_vm, _mb_id, _state) … #define virtio_mem_sbm_for_each_mb_rev(_vm, _mb_id, _state) … /* * Calculate the bit number in the subblock bitmap for the given subblock * inside the given memory block. */ static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm, unsigned long mb_id, int sb_id) { … } /* * Mark all selected subblocks plugged. * * Will not modify the state of the memory block. */ static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Mark all selected subblocks unplugged. * * Will not modify the state of the memory block. */ static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Test if all selected subblocks are plugged. */ static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Test if all selected subblocks are unplugged. */ static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Find the first unplugged subblock. Returns vm->sbm.sbs_per_mb in case there is * none. */ static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * Prepare the subblock bitmap for the next memory block. */ static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm) { … } /* * Test if we could add memory without creating too much offline memory - * to avoid running OOM if memory is getting onlined deferred. */ static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size) { … } /* * Try adding memory to Linux. Will usually only fail if out of memory. * * Must not be called with the vm->hotplug_mutex held (possible deadlock with * onlining code). * * Will not modify the state of memory blocks in virtio-mem. */ static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } /* * See virtio_mem_add_memory(): Try adding a single Linux memory block. */ static int virtio_mem_sbm_add_mb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * See virtio_mem_add_memory(): Try adding a big block. */ static int virtio_mem_bbm_add_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Try removing memory from Linux. Will only fail if memory blocks aren't * offline. * * Must not be called with the vm->hotplug_mutex held (possible deadlock with * onlining code). * * Will not modify the state of memory blocks in virtio-mem. */ static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } /* * See virtio_mem_remove_memory(): Try removing a single Linux memory block. */ static int virtio_mem_sbm_remove_mb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * Try offlining and removing memory from Linux. * * Must not be called with the vm->hotplug_mutex held (possible deadlock with * onlining code). * * Will not modify the state of memory blocks in virtio-mem. */ static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } /* * See virtio_mem_offline_and_remove_memory(): Try offlining and removing * a single Linux memory block. */ static int virtio_mem_sbm_offline_and_remove_mb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * Try (offlining and) removing memory from Linux in case all subblocks are * unplugged. Can be called on online and offline memory blocks. * * May modify the state of memory blocks in virtio-mem. */ static int virtio_mem_sbm_try_remove_unplugged_mb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * See virtio_mem_offline_and_remove_memory(): Try to offline and remove a * all Linux memory blocks covered by the big block. */ static int virtio_mem_bbm_offline_and_remove_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Trigger the workqueue so the device can perform its magic. */ static void virtio_mem_retry(struct virtio_mem *vm) { … } static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id) { … } /* * Test if a virtio-mem device overlaps with the given range. Can be called * from (notifier) callbacks lockless. */ static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start, uint64_t size) { … } /* * Test if a virtio-mem device contains a given range. Can be called from * (notifier) callbacks lockless. */ static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start, uint64_t size) { … } static int virtio_mem_sbm_notify_going_online(struct virtio_mem *vm, unsigned long mb_id) { … } static void virtio_mem_sbm_notify_offline(struct virtio_mem *vm, unsigned long mb_id) { … } static void virtio_mem_sbm_notify_online(struct virtio_mem *vm, unsigned long mb_id, unsigned long start_pfn) { … } static void virtio_mem_sbm_notify_going_offline(struct virtio_mem *vm, unsigned long mb_id) { … } static void virtio_mem_sbm_notify_cancel_offline(struct virtio_mem *vm, unsigned long mb_id) { … } static void virtio_mem_bbm_notify_going_offline(struct virtio_mem *vm, unsigned long bb_id, unsigned long pfn, unsigned long nr_pages) { … } static void virtio_mem_bbm_notify_cancel_offline(struct virtio_mem *vm, unsigned long bb_id, unsigned long pfn, unsigned long nr_pages) { … } /* * This callback will either be called synchronously from add_memory() or * asynchronously (e.g., triggered via user space). We have to be careful * with locking when calling add_memory(). */ static int virtio_mem_memory_notifier_cb(struct notifier_block *nb, unsigned long action, void *arg) { … } static int virtio_mem_pm_notifier_cb(struct notifier_block *nb, unsigned long action, void *arg) { … } /* * Set a range of pages PG_offline. Remember pages that were never onlined * (via generic_online_page()) using PageDirty(). */ static void virtio_mem_set_fake_offline(unsigned long pfn, unsigned long nr_pages, bool onlined) { … } /* * Clear PG_offline from a range of pages. If the pages were never onlined, * (via generic_online_page()), clear PageDirty(). */ static void virtio_mem_clear_fake_offline(unsigned long pfn, unsigned long nr_pages, bool onlined) { … } /* * Release a range of fake-offline pages to the buddy, effectively * fake-onlining them. */ static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages) { … } /* * Try to allocate a range, marking pages fake-offline, effectively * fake-offlining them. */ static int virtio_mem_fake_offline(struct virtio_mem *vm, unsigned long pfn, unsigned long nr_pages) { … } /* * Handle fake-offline pages when memory is going offline - such that the * pages can be skipped by mm-core when offlining. */ static void virtio_mem_fake_offline_going_offline(unsigned long pfn, unsigned long nr_pages) { … } /* * Handle fake-offline pages when memory offlining is canceled - to undo * what we did in virtio_mem_fake_offline_going_offline(). */ static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn, unsigned long nr_pages) { … } static void virtio_mem_online_page(struct virtio_mem *vm, struct page *page, unsigned int order) { … } static void virtio_mem_online_page_cb(struct page *page, unsigned int order) { … } static uint64_t virtio_mem_send_request(struct virtio_mem *vm, const struct virtio_mem_req *req) { … } static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm) { … } /* * Plug selected subblocks. Updates the plugged state, but not the state * of the memory block. */ static int virtio_mem_sbm_plug_sb(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Unplug selected subblocks. Updates the plugged state, but not the state * of the memory block. */ static int virtio_mem_sbm_unplug_sb(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Request to unplug a big block. * * Will not modify the state of the big block. */ static int virtio_mem_bbm_unplug_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Request to plug a big block. * * Will not modify the state of the big block. */ static int virtio_mem_bbm_plug_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Unplug the desired number of plugged subblocks of a offline or not-added * memory block. Will fail if any subblock cannot get unplugged (instead of * skipping it). * * Will not modify the state of the memory block. * * Note: can fail after some subblocks were unplugged. */ static int virtio_mem_sbm_unplug_any_sb_raw(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } /* * Unplug all plugged subblocks of an offline or not-added memory block. * * Will not modify the state of the memory block. * * Note: can fail after some subblocks were unplugged. */ static int virtio_mem_sbm_unplug_mb(struct virtio_mem *vm, unsigned long mb_id) { … } /* * Prepare tracking data for the next memory block. */ static int virtio_mem_sbm_prepare_next_mb(struct virtio_mem *vm, unsigned long *mb_id) { … } /* * Try to plug the desired number of subblocks and add the memory block * to Linux. * * Will modify the state of the memory block. */ static int virtio_mem_sbm_plug_and_add_mb(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } /* * Try to plug the desired number of subblocks of a memory block that * is already added to Linux. * * Will modify the state of the memory block. * * Note: Can fail after some subblocks were successfully plugged. */ static int virtio_mem_sbm_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } static int virtio_mem_sbm_plug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Plug a big block and add it to Linux. * * Will modify the state of the big block. */ static int virtio_mem_bbm_plug_and_add_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Prepare tracking data for the next big block. */ static int virtio_mem_bbm_prepare_next_bb(struct virtio_mem *vm, unsigned long *bb_id) { … } static int virtio_mem_bbm_plug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Try to plug the requested amount of memory. */ static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Unplug the desired number of plugged subblocks of an offline memory block. * Will fail if any subblock cannot get unplugged (instead of skipping it). * * Will modify the state of the memory block. Might temporarily drop the * hotplug_mutex. * * Note: Can fail after some subblocks were successfully unplugged. */ static int virtio_mem_sbm_unplug_any_sb_offline(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } /* * Unplug the given plugged subblocks of an online memory block. * * Will modify the state of the memory block. */ static int virtio_mem_sbm_unplug_sb_online(struct virtio_mem *vm, unsigned long mb_id, int sb_id, int count) { … } /* * Unplug the desired number of plugged subblocks of an online memory block. * Will skip subblock that are busy. * * Will modify the state of the memory block. Might temporarily drop the * hotplug_mutex. * * Note: Can fail after some subblocks were successfully unplugged. Can * return 0 even if subblocks were busy and could not get unplugged. */ static int virtio_mem_sbm_unplug_any_sb_online(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } /* * Unplug the desired number of plugged subblocks of a memory block that is * already added to Linux. Will skip subblock of online memory blocks that are * busy (by the OS). Will fail if any subblock that's not busy cannot get * unplugged. * * Will modify the state of the memory block. Might temporarily drop the * hotplug_mutex. * * Note: Can fail after some subblocks were successfully unplugged. Can * return 0 even if subblocks were busy and could not get unplugged. */ static int virtio_mem_sbm_unplug_any_sb(struct virtio_mem *vm, unsigned long mb_id, uint64_t *nb_sb) { … } static int virtio_mem_sbm_unplug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Try to offline and remove a big block from Linux and unplug it. Will fail * with -EBUSY if some memory is busy and cannot get unplugged. * * Will modify the state of the memory block. Might temporarily drop the * hotplug_mutex. */ static int virtio_mem_bbm_offline_remove_and_unplug_bb(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Test if a big block is completely offline. */ static bool virtio_mem_bbm_bb_is_offline(struct virtio_mem *vm, unsigned long bb_id) { … } /* * Test if a big block is completely onlined to ZONE_MOVABLE (or offline). */ static bool virtio_mem_bbm_bb_is_movable(struct virtio_mem *vm, unsigned long bb_id) { … } static int virtio_mem_bbm_unplug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Try to unplug the requested amount of memory. */ static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff) { … } /* * Try to unplug all blocks that couldn't be unplugged before, for example, * because the hypervisor was busy. Further, offline and remove any memory * blocks where we previously failed. */ static int virtio_mem_cleanup_pending_mb(struct virtio_mem *vm) { … } /* * Update all parts of the config that could have changed. */ static void virtio_mem_refresh_config(struct virtio_mem *vm) { … } /* * Workqueue function for handling plug/unplug requests and config updates. */ static void virtio_mem_run_wq(struct work_struct *work) { … } static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer) { … } static void virtio_mem_handle_response(struct virtqueue *vq) { … } static int virtio_mem_init_vq(struct virtio_mem *vm) { … } static int virtio_mem_init_hotplug(struct virtio_mem *vm) { … } #ifdef CONFIG_PROC_VMCORE static int virtio_mem_send_state_request(struct virtio_mem *vm, uint64_t addr, uint64_t size) { … } static bool virtio_mem_vmcore_pfn_is_ram(struct vmcore_cb *cb, unsigned long pfn) { … } #endif /* CONFIG_PROC_VMCORE */ static int virtio_mem_init_kdump(struct virtio_mem *vm) { … } static int virtio_mem_init(struct virtio_mem *vm) { … } static int virtio_mem_create_resource(struct virtio_mem *vm) { … } static void virtio_mem_delete_resource(struct virtio_mem *vm) { … } static int virtio_mem_range_has_system_ram(struct resource *res, void *arg) { … } static bool virtio_mem_has_memory_added(struct virtio_mem *vm) { … } static int virtio_mem_probe(struct virtio_device *vdev) { … } static void virtio_mem_deinit_hotplug(struct virtio_mem *vm) { … } static void virtio_mem_deinit_kdump(struct virtio_mem *vm) { … } static void virtio_mem_remove(struct virtio_device *vdev) { … } static void virtio_mem_config_changed(struct virtio_device *vdev) { … } #ifdef CONFIG_PM_SLEEP static int virtio_mem_freeze(struct virtio_device *vdev) { … } static int virtio_mem_restore(struct virtio_device *vdev) { … } #endif static unsigned int virtio_mem_features[] = …; static const struct virtio_device_id virtio_mem_id_table[] = …; static struct virtio_driver virtio_mem_driver = …; module_virtio_driver(…) …; MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table); MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux