// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2023 Red Hat */ /* * This file contains the main entry points for normal operations on a vdo as well as functions for * constructing and destroying vdo instances (in memory). */ /** * DOC: * * A read_only_notifier has a single completion which is used to perform read-only notifications, * however, vdo_enter_read_only_mode() may be called from any thread. A pair of fields, protected * by a spinlock, are used to control the read-only mode entry process. The first field holds the * read-only error. The second is the state field, which may hold any of the four special values * enumerated here. * * When vdo_enter_read_only_mode() is called from some vdo thread, if the read_only_error field * already contains an error (i.e. its value is not VDO_SUCCESS), then some other error has already * initiated the read-only process, and nothing more is done. Otherwise, the new error is stored in * the read_only_error field, and the state field is consulted. If the state is MAY_NOTIFY, it is * set to NOTIFYING, and the notification process begins. If the state is MAY_NOT_NOTIFY, then * notifications are currently disallowed, generally due to the vdo being suspended. In this case, * the nothing more will be done until the vdo is resumed, at which point the notification will be * performed. In any other case, the vdo is already read-only, and there is nothing more to do. */ #include "vdo.h" #include <linux/completion.h> #include <linux/device-mapper.h> #include <linux/kernel.h> #include <linux/lz4.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/types.h> #include "logger.h" #include "memory-alloc.h" #include "permassert.h" #include "string-utils.h" #include "block-map.h" #include "completion.h" #include "data-vio.h" #include "dedupe.h" #include "encodings.h" #include "funnel-workqueue.h" #include "io-submitter.h" #include "logical-zone.h" #include "packer.h" #include "physical-zone.h" #include "recovery-journal.h" #include "slab-depot.h" #include "statistics.h" #include "status-codes.h" #include "vio.h" #define PARANOID_THREAD_CONSISTENCY_CHECKS … struct sync_completion { … }; /* A linked list is adequate for the small number of entries we expect. */ struct device_registry { … }; static struct device_registry registry; /** * vdo_initialize_device_registry_once() - Initialize the necessary structures for the device * registry. */ void vdo_initialize_device_registry_once(void) { … } /** vdo_is_equal() - Implements vdo_filter_fn. */ static bool vdo_is_equal(struct vdo *vdo, const void *context) { … } /** * filter_vdos_locked() - Find a vdo in the registry if it exists there. * @filter: The filter function to apply to devices. * @context: A bit of context to provide the filter. * * Context: Must be called holding the lock. * * Return: the vdo object found, if any. */ static struct vdo * __must_check filter_vdos_locked(vdo_filter_fn filter, const void *context) { … } /** * vdo_find_matching() - Find and return the first (if any) vdo matching a given filter function. * @filter: The filter function to apply to vdos. * @context: A bit of context to provide the filter. */ struct vdo *vdo_find_matching(vdo_filter_fn filter, const void *context) { … } static void start_vdo_request_queue(void *ptr) { … } static void finish_vdo_request_queue(void *ptr) { … } #ifdef MODULE #define MODULE_NAME … #else #define MODULE_NAME … #endif /* MODULE */ static const struct vdo_work_queue_type default_queue_type = …; static const struct vdo_work_queue_type bio_ack_q_type = …; static const struct vdo_work_queue_type cpu_q_type = …; static void uninitialize_thread_config(struct thread_config *config) { … } static void assign_thread_ids(struct thread_config *config, thread_id_t thread_ids[], zone_count_t count) { … } /** * initialize_thread_config() - Initialize the thread mapping * * If the logical, physical, and hash zone counts are all 0, a single thread will be shared by all * three plus the packer and recovery journal. Otherwise, there must be at least one of each type, * and each will have its own thread, as will the packer and recovery journal. * * Return: VDO_SUCCESS or an error. */ static int __must_check initialize_thread_config(struct thread_count_config counts, struct thread_config *config) { … } /** * read_geometry_block() - Synchronously read the geometry block from a vdo's underlying block * device. * @vdo: The vdo whose geometry is to be read. * * Return: VDO_SUCCESS or an error code. */ static int __must_check read_geometry_block(struct vdo *vdo) { … } static bool get_zone_thread_name(const thread_id_t thread_ids[], zone_count_t count, thread_id_t id, const char *prefix, char *buffer, size_t buffer_length) { … } /** * get_thread_name() - Format the name of the worker thread desired to support a given work queue. * @thread_config: The thread configuration. * @thread_id: The thread id. * @buffer: Where to put the formatted name. * @buffer_length: Size of the output buffer. * * The physical layer may add a prefix identifying the product; the output from this function * should just identify the thread. */ static void get_thread_name(const struct thread_config *thread_config, thread_id_t thread_id, char *buffer, size_t buffer_length) { … } /** * vdo_make_thread() - Construct a single vdo work_queue and its associated thread (or threads for * round-robin queues). * @vdo: The vdo which owns the thread. * @thread_id: The id of the thread to create (as determined by the thread_config). * @type: The description of the work queue for this thread. * @queue_count: The number of actual threads/queues contained in the "thread". * @contexts: An array of queue_count contexts, one for each individual queue; may be NULL. * * Each "thread" constructed by this method is represented by a unique thread id in the thread * config, and completions can be enqueued to the queue and run on the threads comprising this * entity. * * Return: VDO_SUCCESS or an error. */ int vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, const struct vdo_work_queue_type *type, unsigned int queue_count, void *contexts[]) { … } /** * register_vdo() - Register a VDO; it must not already be registered. * @vdo: The vdo to register. * * Return: VDO_SUCCESS or an error. */ static int register_vdo(struct vdo *vdo) { … } /** * initialize_vdo() - Do the portion of initializing a vdo which will clean up after itself on * error. * @vdo: The vdo being initialized * @config: The configuration of the vdo * @instance: The instance number of the vdo * @reason: The buffer to hold the failure reason on error */ static int initialize_vdo(struct vdo *vdo, struct device_config *config, unsigned int instance, char **reason) { … } /** * vdo_make() - Allocate and initialize a vdo. * @instance: Device instantiation counter. * @config: The device configuration. * @reason: The reason for any failure during this call. * @vdo_ptr: A pointer to hold the created vdo. * * Return: VDO_SUCCESS or an error. */ int vdo_make(unsigned int instance, struct device_config *config, char **reason, struct vdo **vdo_ptr) { … } static void finish_vdo(struct vdo *vdo) { … } /** * free_listeners() - Free the list of read-only listeners associated with a thread. * @thread_data: The thread holding the list to free. */ static void free_listeners(struct vdo_thread *thread) { … } static void uninitialize_super_block(struct vdo_super_block *super_block) { … } /** * unregister_vdo() - Remove a vdo from the device registry. * @vdo: The vdo to remove. */ static void unregister_vdo(struct vdo *vdo) { … } /** * vdo_destroy() - Destroy a vdo instance. * @vdo: The vdo to destroy (may be NULL). */ void vdo_destroy(struct vdo *vdo) { … } static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super_block) { … } /** * finish_reading_super_block() - Continue after loading the super block. * @completion: The super block vio. * * This callback is registered in vdo_load_super_block(). */ static void finish_reading_super_block(struct vdo_completion *completion) { … } /** * handle_super_block_read_error() - Handle an error reading the super block. * @completion: The super block vio. * * This error handler is registered in vdo_load_super_block(). */ static void handle_super_block_read_error(struct vdo_completion *completion) { … } static void read_super_block_endio(struct bio *bio) { … } /** * vdo_load_super_block() - Allocate a super block and read its contents from storage. * @vdo: The vdo containing the super block on disk. * @parent: The completion to notify after loading the super block. */ void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent) { … } /** * vdo_get_backing_device() - Get the block device object underlying a vdo. * @vdo: The vdo. * * Return: The vdo's current block device. */ struct block_device *vdo_get_backing_device(const struct vdo *vdo) { … } /** * vdo_get_device_name() - Get the device name associated with the vdo target. * @target: The target device interface. * * Return: The block device name. */ const char *vdo_get_device_name(const struct dm_target *target) { … } /** * vdo_synchronous_flush() - Issue a flush request and wait for it to complete. * @vdo: The vdo. * * Return: VDO_SUCCESS or an error. */ int vdo_synchronous_flush(struct vdo *vdo) { … } /** * vdo_get_state() - Get the current state of the vdo. * @vdo: The vdo. * Context: This method may be called from any thread. * * Return: The current state of the vdo. */ enum vdo_state vdo_get_state(const struct vdo *vdo) { … } /** * vdo_set_state() - Set the current state of the vdo. * @vdo: The vdo whose state is to be set. * @state: The new state of the vdo. * * Context: This method may be called from any thread. */ void vdo_set_state(struct vdo *vdo, enum vdo_state state) { … } /** * vdo_get_admin_state() - Get the admin state of the vdo. * @vdo: The vdo. * * Return: The code for the vdo's current admin state. */ const struct admin_state_code *vdo_get_admin_state(const struct vdo *vdo) { … } /** * record_vdo() - Record the state of the VDO for encoding in the super block. */ static void record_vdo(struct vdo *vdo) { … } /** * continue_super_block_parent() - Continue the parent of a super block save operation. * @completion: The super block vio. * * This callback is registered in vdo_save_components(). */ static void continue_super_block_parent(struct vdo_completion *completion) { … } /** * handle_save_error() - Log a super block save error. * @completion: The super block vio. * * This error handler is registered in vdo_save_components(). */ static void handle_save_error(struct vdo_completion *completion) { … } static void super_block_write_endio(struct bio *bio) { … } /** * vdo_save_components() - Encode the vdo and save the super block asynchronously. * @vdo: The vdo whose state is being saved. * @parent: The completion to notify when the save is complete. */ void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent) { … } /** * vdo_register_read_only_listener() - Register a listener to be notified when the VDO goes * read-only. * @vdo: The vdo to register with. * @listener: The object to notify. * @notification: The function to call to send the notification. * @thread_id: The id of the thread on which to send the notification. * * Return: VDO_SUCCESS or an error. */ int vdo_register_read_only_listener(struct vdo *vdo, void *listener, vdo_read_only_notification_fn notification, thread_id_t thread_id) { … } /** * notify_vdo_of_read_only_mode() - Notify a vdo that it is going read-only. * @listener: The vdo. * @parent: The completion to notify in order to acknowledge the notification. * * This will save the read-only state to the super block. * * Implements vdo_read_only_notification_fn. */ static void notify_vdo_of_read_only_mode(void *listener, struct vdo_completion *parent) { … } /** * vdo_enable_read_only_entry() - Enable a vdo to enter read-only mode on errors. * @vdo: The vdo to enable. * * Return: VDO_SUCCESS or an error. */ int vdo_enable_read_only_entry(struct vdo *vdo) { … } /** * vdo_wait_until_not_entering_read_only_mode() - Wait until no read-only notifications are in * progress and prevent any subsequent * notifications. * @parent: The completion to notify when no threads are entering read-only mode. * * Notifications may be re-enabled by calling vdo_allow_read_only_mode_entry(). */ void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent) { … } /** * as_notifier() - Convert a generic vdo_completion to a read_only_notifier. * @completion: The completion to convert. * * Return: The completion as a read_only_notifier. */ static inline struct read_only_notifier *as_notifier(struct vdo_completion *completion) { … } /** * finish_entering_read_only_mode() - Complete the process of entering read only mode. * @completion: The read-only mode completion. */ static void finish_entering_read_only_mode(struct vdo_completion *completion) { … } /** * make_thread_read_only() - Inform each thread that the VDO is in read-only mode. * @completion: The read-only mode completion. */ static void make_thread_read_only(struct vdo_completion *completion) { … } /** * vdo_allow_read_only_mode_entry() - Allow the notifier to put the VDO into read-only mode, * reversing the effects of * vdo_wait_until_not_entering_read_only_mode(). * @parent: The object to notify once the operation is complete. * * If some thread tried to put the vdo into read-only mode while notifications were disallowed, it * will be done when this method is called. If that happens, the parent will not be notified until * the vdo has actually entered read-only mode and attempted to save the super block. * * Context: This method may only be called from the admin thread. */ void vdo_allow_read_only_mode_entry(struct vdo_completion *parent) { … } /** * vdo_enter_read_only_mode() - Put a VDO into read-only mode and save the read-only state in the * super block. * @vdo: The vdo. * @error_code: The error which caused the VDO to enter read-only mode. * * This method is a no-op if the VDO is already read-only. */ void vdo_enter_read_only_mode(struct vdo *vdo, int error_code) { … } /** * vdo_is_read_only() - Check whether the VDO is read-only. * @vdo: The vdo. * * Return: true if the vdo is read-only. * * This method may be called from any thread, as opposed to examining the VDO's state field which * is only safe to check from the admin thread. */ bool vdo_is_read_only(struct vdo *vdo) { … } /** * vdo_in_read_only_mode() - Check whether a vdo is in read-only mode. * @vdo: The vdo to query. * * Return: true if the vdo is in read-only mode. */ bool vdo_in_read_only_mode(const struct vdo *vdo) { … } /** * vdo_in_recovery_mode() - Check whether the vdo is in recovery mode. * @vdo: The vdo to query. * * Return: true if the vdo is in recovery mode. */ bool vdo_in_recovery_mode(const struct vdo *vdo) { … } /** * vdo_enter_recovery_mode() - Put the vdo into recovery mode. * @vdo: The vdo. */ void vdo_enter_recovery_mode(struct vdo *vdo) { … } /** * complete_synchronous_action() - Signal the waiting thread that a synchronous action is complete. * @completion: The sync completion. */ static void complete_synchronous_action(struct vdo_completion *completion) { … } /** * perform_synchronous_action() - Launch an action on a VDO thread and wait for it to complete. * @vdo: The vdo. * @action: The callback to launch. * @thread_id: The thread on which to run the action. * @parent: The parent of the sync completion (may be NULL). */ static int perform_synchronous_action(struct vdo *vdo, vdo_action_fn action, thread_id_t thread_id, void *parent) { … } /** * set_compression_callback() - Callback to turn compression on or off. * @completion: The completion. */ static void set_compression_callback(struct vdo_completion *completion) { … } /** * vdo_set_compressing() - Turn compression on or off. * @vdo: The vdo. * @enable: Whether to enable or disable compression. * * Return: Whether compression was previously on or off. */ bool vdo_set_compressing(struct vdo *vdo, bool enable) { … } /** * vdo_get_compressing() - Get whether compression is enabled in a vdo. * @vdo: The vdo. * * Return: State of compression. */ bool vdo_get_compressing(struct vdo *vdo) { … } static size_t get_block_map_cache_size(const struct vdo *vdo) { … } static struct error_statistics __must_check get_vdo_error_statistics(const struct vdo *vdo) { … } static void copy_bio_stat(struct bio_stats *b, const struct atomic_bio_stats *a) { … } static struct bio_stats subtract_bio_stats(struct bio_stats minuend, struct bio_stats subtrahend) { … } /** * vdo_get_physical_blocks_allocated() - Get the number of physical blocks in use by user data. * @vdo: The vdo. * * Return: The number of blocks allocated for user data. */ static block_count_t __must_check vdo_get_physical_blocks_allocated(const struct vdo *vdo) { … } /** * vdo_get_physical_blocks_overhead() - Get the number of physical blocks used by vdo metadata. * @vdo: The vdo. * * Return: The number of overhead blocks. */ static block_count_t __must_check vdo_get_physical_blocks_overhead(const struct vdo *vdo) { … } static const char *vdo_describe_state(enum vdo_state state) { … } /** * get_vdo_statistics() - Populate a vdo_statistics structure on the admin thread. * @vdo: The vdo. * @stats: The statistics structure to populate. */ static void get_vdo_statistics(const struct vdo *vdo, struct vdo_statistics *stats) { … } /** * vdo_fetch_statistics_callback() - Action to populate a vdo_statistics * structure on the admin thread. * @completion: The completion. * * This callback is registered in vdo_fetch_statistics(). */ static void vdo_fetch_statistics_callback(struct vdo_completion *completion) { … } /** * vdo_fetch_statistics() - Fetch statistics on the correct thread. * @vdo: The vdo. * @stats: The vdo statistics are returned here. */ void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats) { … } /** * vdo_get_callback_thread_id() - Get the id of the callback thread on which a completion is * currently running. * * Return: The current thread ID, or -1 if no such thread. */ thread_id_t vdo_get_callback_thread_id(void) { … } /** * vdo_dump_status() - Dump status information about a vdo to the log for debugging. * @vdo: The vdo to dump. */ void vdo_dump_status(const struct vdo *vdo) { … } /** * vdo_assert_on_admin_thread() - Assert that we are running on the admin thread. * @vdo: The vdo. * @name: The name of the function which should be running on the admin thread (for logging). */ void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name) { … } /** * vdo_assert_on_logical_zone_thread() - Assert that this function was called on the specified * logical zone thread. * @vdo: The vdo. * @logical_zone: The number of the logical zone. * @name: The name of the calling function. */ void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, const char *name) { … } /** * vdo_assert_on_physical_zone_thread() - Assert that this function was called on the specified * physical zone thread. * @vdo: The vdo. * @physical_zone: The number of the physical zone. * @name: The name of the calling function. */ void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone, const char *name) { … } /** * vdo_get_physical_zone() - Get the physical zone responsible for a given physical block number. * @vdo: The vdo containing the physical zones. * @pbn: The PBN of the data block. * @zone_ptr: A pointer to return the physical zone. * * Gets the physical zone responsible for a given physical block number of a data block in this vdo * instance, or of the zero block (for which a NULL zone is returned). For any other block number * that is not in the range of valid data block numbers in any slab, an error will be returned. * This function is safe to call on invalid block numbers; it will not put the vdo into read-only * mode. * * Return: VDO_SUCCESS or VDO_OUT_OF_RANGE if the block number is invalid or an error code for any * other failure. */ int vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, struct physical_zone **zone_ptr) { … }
Codebase Browser
linux