// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2023 Red Hat */ #include "recovery-journal.h" #include <linux/atomic.h> #include <linux/bio.h> #include "logger.h" #include "memory-alloc.h" #include "permassert.h" #include "block-map.h" #include "completion.h" #include "constants.h" #include "data-vio.h" #include "encodings.h" #include "io-submitter.h" #include "slab-depot.h" #include "types.h" #include "vdo.h" #include "vio.h" #include "wait-queue.h" static const u64 RECOVERY_COUNT_MASK = …; /* * The number of reserved blocks must be large enough to prevent a new recovery journal * block write from overwriting a block which appears to still be a valid head block of the * journal. Currently, that means reserving enough space for all 2048 data_vios. */ #define RECOVERY_JOURNAL_RESERVED_BLOCKS … /** * DOC: Lock Counters. * * A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The * per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0 * is at index 0, zone 0's lock 1 is at index 1, and zone 1's lock 0 is at index 'locks'. This * arrangement is intended to minimize cache-line contention for counters from different zones. * * The locks are implemented as a single object instead of as a lock counter per lock both to * afford this opportunity to reduce cache line contention and also to eliminate the need to have a * completion per lock. * * Lock sets are laid out with the set for recovery journal first, followed by the logical zones, * and then the physical zones. */ enum lock_counter_state { … }; /** * get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone. * @journal: The recovery journal. * @lock_number: The lock to get. * @zone_type: The zone type whose count is desired. * * Return: A pointer to the zone count for the given lock and zone. */ static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal, block_count_t lock_number, enum vdo_zone_type zone_type) { … } /** * get_counter() - Get the zone counter for a given lock on a given zone. * @journal: The recovery journal. * @lock_number: The lock to get. * @zone_type: The zone type whose count is desired. * @zone_id: The zone index whose count is desired. * * Return: The counter for the given lock and zone. */ static inline u16 *get_counter(struct recovery_journal *journal, block_count_t lock_number, enum vdo_zone_type zone_type, zone_count_t zone_id) { … } static atomic_t *get_decrement_counter(struct recovery_journal *journal, block_count_t lock_number) { … } /** * is_journal_zone_locked() - Check whether the journal zone is locked for a given lock. * @journal: The recovery journal. * @lock_number: The lock to check. * * Return: true if the journal zone is locked. */ static bool is_journal_zone_locked(struct recovery_journal *journal, block_count_t lock_number) { … } /** * vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal * block. * @journal: The recovery journal. * @sequence_number: The journal sequence number of the referenced block. * @zone_type: The type of the zone making the adjustment. * @zone_id: The ID of the zone making the adjustment. * * If this is the last reference for a given zone type, an attempt will be made to reap the * journal. */ void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal, sequence_number_t sequence_number, enum vdo_zone_type zone_type, zone_count_t zone_id) { … } static inline struct recovery_journal_block * __must_check get_journal_block(struct list_head *list) { … } /** * pop_free_list() - Get a block from the end of the free list. * @journal: The journal. * * Return: The block or NULL if the list is empty. */ static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal) { … } /** * is_block_dirty() - Check whether a recovery block is dirty. * @block: The block to check. * * Indicates it has any uncommitted entries, which includes both entries not written and entries * written but not yet acknowledged. * * Return: true if the block has any uncommitted entries. */ static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block) { … } /** * is_block_empty() - Check whether a journal block is empty. * @block: The block to check. * * Return: true if the block has no entries. */ static inline bool __must_check is_block_empty(const struct recovery_journal_block *block) { … } /** * is_block_full() - Check whether a journal block is full. * @block: The block to check. * * Return: true if the block is full. */ static inline bool __must_check is_block_full(const struct recovery_journal_block *block) { … } /** * assert_on_journal_thread() - Assert that we are running on the journal thread. * @journal: The journal. * @function_name: The function doing the check (for logging). */ static void assert_on_journal_thread(struct recovery_journal *journal, const char *function_name) { … } /** * continue_waiter() - Release a data_vio from the journal. * * Invoked whenever a data_vio is to be released from the journal, either because its entry was * committed to disk, or because there was an error. Implements waiter_callback_fn. */ static void continue_waiter(struct vdo_waiter *waiter, void *context) { … } /** * has_block_waiters() - Check whether the journal has any waiters on any blocks. * @journal: The journal in question. * * Return: true if any block has a waiter. */ static inline bool has_block_waiters(struct recovery_journal *journal) { … } static void recycle_journal_blocks(struct recovery_journal *journal); static void recycle_journal_block(struct recovery_journal_block *block); static void notify_commit_waiters(struct recovery_journal *journal); /** * suspend_lock_counter() - Prevent the lock counter from notifying. * @counter: The counter. * * Return: true if the lock counter was not notifying and hence the suspend was efficacious. */ static bool suspend_lock_counter(struct lock_counter *counter) { … } static inline bool is_read_only(struct recovery_journal *journal) { … } /** * check_for_drain_complete() - Check whether the journal has drained. * @journal: The journal which may have just drained. */ static void check_for_drain_complete(struct recovery_journal *journal) { … } /** * notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone * read-only. * @listener: The journal. * @parent: The completion to notify in order to acknowledge the notification. * * Implements vdo_read_only_notification_fn. */ static void notify_recovery_journal_of_read_only_mode(void *listener, struct vdo_completion *parent) { … } /** * enter_journal_read_only_mode() - Put the journal in read-only mode. * @journal: The journal which has failed. * @error_code: The error result triggering this call. * * All attempts to add entries after this function is called will fail. All VIOs waiting for * commits will be awakened with an error. */ static void enter_journal_read_only_mode(struct recovery_journal *journal, int error_code) { … } /** * vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current * sequence number. * @journal: The journal in question. * * Exposed only so the block map can be initialized therefrom. * * Return: The sequence number of the tail block. */ sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal) { … } /** * get_recovery_journal_head() - Get the head of the recovery journal. * @journal: The journal. * * The head is the lowest sequence number of the block map head and the slab journal head. * * Return: the head of the journal. */ static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal) { … } /** * compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count. * @recovery_count: The recovery count. * * Return: The byte corresponding to the recovery count. */ static inline u8 __must_check compute_recovery_count_byte(u64 recovery_count) { … } /** * check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if * so, force the oldest slab journal tail block to commit. * @journal: The journal. */ static void check_slab_journal_commit_threshold(struct recovery_journal *journal) { … } static void reap_recovery_journal(struct recovery_journal *journal); static void assign_entries(struct recovery_journal *journal); /** * finish_reaping() - Finish reaping the journal. * @journal: The journal being reaped. */ static void finish_reaping(struct recovery_journal *journal) { … } /** * complete_reaping() - Finish reaping the journal after flushing the lower layer. * @completion: The journal's flush VIO. * * This is the callback registered in reap_recovery_journal(). */ static void complete_reaping(struct vdo_completion *completion) { … } /** * handle_flush_error() - Handle an error when flushing the lower layer due to reaping. * @completion: The journal's flush VIO. */ static void handle_flush_error(struct vdo_completion *completion) { … } static void flush_endio(struct bio *bio) { … } /** * initialize_journal_state() - Set all journal fields appropriately to start journaling from the * current active block. * @journal: The journal to be reset based on its active block. */ static void initialize_journal_state(struct recovery_journal *journal) { … } /** * vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks. * @journal_size: The size of the recovery journal in blocks. * * Return: the number of recovery journal blocks usable for entries. */ block_count_t vdo_get_recovery_journal_length(block_count_t journal_size) { … } /** * reap_recovery_journal_callback() - Attempt to reap the journal. * @completion: The lock counter completion. * * Attempts to reap the journal now that all the locks on some journal block have been released. * This is the callback registered with the lock counter. */ static void reap_recovery_journal_callback(struct vdo_completion *completion) { … } /** * initialize_lock_counter() - Initialize a lock counter. * * @journal: The recovery journal. * @vdo: The vdo. * * Return: VDO_SUCCESS or an error. */ static int __must_check initialize_lock_counter(struct recovery_journal *journal, struct vdo *vdo) { … } /** * set_journal_tail() - Set the journal's tail sequence number. * @journal: The journal whose tail is to be set. * @tail: The new tail value. */ static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail) { … } /** * initialize_recovery_block() - Initialize a journal block. * @vdo: The vdo from which to construct vios. * @journal: The journal to which the block will belong. * @block: The block to initialize. * * Return: VDO_SUCCESS or an error. */ static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal, struct recovery_journal_block *block) { … } /** * vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that * was decoded from the super block. * * @state: The decoded state of the journal. * @nonce: The nonce of the VDO. * @vdo: The VDO. * @partition: The partition for the journal. * @recovery_count: The VDO's number of completed recoveries. * @journal_size: The number of blocks in the journal on disk. * @journal_ptr: The pointer to hold the new recovery journal. * * Return: A success or error code. */ int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t nonce, struct vdo *vdo, struct partition *partition, u64 recovery_count, block_count_t journal_size, struct recovery_journal **journal_ptr) { … } /** * vdo_free_recovery_journal() - Free a recovery journal. * @journal: The recovery journal to free. */ void vdo_free_recovery_journal(struct recovery_journal *journal) { … } /** * vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair. * @journal: The journal in question. * @recovery_count: The number of completed recoveries. * @tail: The new tail block sequence number. * @logical_blocks_used: The new number of logical blocks used. * @block_map_data_blocks: The new number of block map data blocks. */ void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal, u64 recovery_count, sequence_number_t tail, block_count_t logical_blocks_used, block_count_t block_map_data_blocks) { … } /** * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from * data blocks, currently in use. * @journal: The journal in question. * * Return: The number of block map pages allocated from slabs. */ block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal) { … } /** * vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread. * @journal: The journal to query. * * Return: The ID of the journal's thread. */ thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal) { … } /** * vdo_open_recovery_journal() - Prepare the journal for new entries. * @journal: The journal in question. * @depot: The slab depot for this VDO. * @block_map: The block map for this VDO. */ void vdo_open_recovery_journal(struct recovery_journal *journal, struct slab_depot *depot, struct block_map *block_map) { … } /** * vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super * block. * @journal: the recovery journal. * * Return: the state of the journal. */ struct recovery_journal_state_7_0 vdo_record_recovery_journal(const struct recovery_journal *journal) { … } /** * get_block_header() - Get a pointer to the packed journal block header in the block buffer. * @block: The recovery block. * * Return: The block's header. */ static inline struct packed_journal_header * get_block_header(const struct recovery_journal_block *block) { … } /** * set_active_sector() - Set the current sector of the current block and initialize it. * @block: The block to update. * @sector: A pointer to the first byte of the new sector. */ static void set_active_sector(struct recovery_journal_block *block, void *sector) { … } /** * advance_tail() - Advance the tail of the journal. * @journal: The journal whose tail should be advanced. * * Return: true if the tail was advanced. */ static bool advance_tail(struct recovery_journal *journal) { … } /** * initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock. * @journal: The recovery journal. * * Context: This must be called from the journal zone. */ static void initialize_lock_count(struct recovery_journal *journal) { … } /** * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check * whether an entry of the given type may be assigned at this time. * @journal: The journal receiving an entry. * * Return: true if there is space in the journal to store an entry of the specified type. */ static bool prepare_to_assign_entry(struct recovery_journal *journal) { … } static void write_blocks(struct recovery_journal *journal); /** * schedule_block_write() - Queue a block for writing. * @journal: The journal in question. * @block: The block which is now ready to write. * * The block is expected to be full. If the block is currently writing, this is a noop as the block * will be queued for writing when the write finishes. The block must not currently be queued for * writing. */ static void schedule_block_write(struct recovery_journal *journal, struct recovery_journal_block *block) { … } /** * release_journal_block_reference() - Release a reference to a journal block. * @block: The journal block from which to release a reference. */ static void release_journal_block_reference(struct recovery_journal_block *block) { … } static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio) { … } /** * assign_entry() - Assign an entry waiter to the active block. * * Implements waiter_callback_fn. */ static void assign_entry(struct vdo_waiter *waiter, void *context) { … } static void assign_entries(struct recovery_journal *journal) { … } /** * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been * fully committed. * @block: The block to be recycled. */ static void recycle_journal_block(struct recovery_journal_block *block) { … } /** * continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because * its entry was committed to disk. * * Implements waiter_callback_fn. */ static void continue_committed_waiter(struct vdo_waiter *waiter, void *context) { … } /** * notify_commit_waiters() - Notify any VIOs whose entries have now committed. * @journal: The recovery journal to update. */ static void notify_commit_waiters(struct recovery_journal *journal) { … } /** * recycle_journal_blocks() - Recycle any journal blocks which have been fully committed. * @journal: The recovery journal to update. */ static void recycle_journal_blocks(struct recovery_journal *journal) { … } /** * complete_write() - Handle post-commit processing. * @completion: The completion of the VIO writing this block. * * This is the callback registered by write_block(). If more entries accumulated in the block being * committed while the commit was in progress, another commit will be initiated. */ static void complete_write(struct vdo_completion *completion) { … } static void handle_write_error(struct vdo_completion *completion) { … } static void complete_write_endio(struct bio *bio) { … } /** * add_queued_recovery_entries() - Actually add entries from the queue to the given block. * @block: The journal block. */ static void add_queued_recovery_entries(struct recovery_journal_block *block) { … } /** * write_block() - Issue a block for writing. * * Implements waiter_callback_fn. */ static void write_block(struct vdo_waiter *waiter, void *context __always_unused) { … } /** * write_blocks() - Attempt to commit blocks, according to write policy. * @journal: The recovery journal. */ static void write_blocks(struct recovery_journal *journal) { … } /** * vdo_add_recovery_journal_entry() - Add an entry to a recovery journal. * @journal: The journal in which to make an entry. * @data_vio: The data_vio for which to add the entry. The entry will be taken * from the logical and new_mapped fields of the data_vio. The * data_vio's recovery_sequence_number field will be set to the * sequence number of the journal block in which the entry was * made. * * This method is asynchronous. The data_vio will not be called back until the entry is committed * to the on-disk journal. */ void vdo_add_recovery_journal_entry(struct recovery_journal *journal, struct data_vio *data_vio) { … } /** * is_lock_locked() - Check whether a lock is locked for a zone type. * @journal: The recovery journal. * @lock_number: The lock to check. * @zone_type: The type of the zone. * * If the recovery journal has a lock on the lock number, both logical and physical zones are * considered locked. * * Return: true if the specified lock has references (is locked). */ static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number, enum vdo_zone_type zone_type) { … } /** * reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks. * @journal: The recovery journal. */ static void reap_recovery_journal(struct recovery_journal *journal) { … } /** * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block * from somewhere other than the journal itself. * @journal: The recovery journal. * @sequence_number: The journal sequence number of the referenced block. * @zone_type: The type of the zone making the adjustment. * @zone_id: The ID of the zone making the adjustment. */ void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal, sequence_number_t sequence_number, enum vdo_zone_type zone_type, zone_count_t zone_id) { … } /** * vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery * journal block. * @journal: The recovery journal. * @sequence_number: The journal sequence number of the referenced block. */ void vdo_release_journal_entry_lock(struct recovery_journal *journal, sequence_number_t sequence_number) { … } /** * initiate_drain() - Initiate a drain. * * Implements vdo_admin_initiator_fn. */ static void initiate_drain(struct admin_state *state) { … } /** * vdo_drain_recovery_journal() - Drain recovery journal I/O. * @journal: The journal to drain. * @operation: The drain operation (suspend or save). * @parent: The completion to notify once the journal is drained. * * All uncommitted entries will be written out. */ void vdo_drain_recovery_journal(struct recovery_journal *journal, const struct admin_state_code *operation, struct vdo_completion *parent) { … } /** * resume_lock_counter() - Re-allow notifications from a suspended lock counter. * @counter: The counter. * * Return: true if the lock counter was suspended. */ static bool resume_lock_counter(struct lock_counter *counter) { … } /** * vdo_resume_recovery_journal() - Resume a recovery journal which has been drained. * @journal: The journal to resume. * @parent: The completion to finish once the journal is resumed. */ void vdo_resume_recovery_journal(struct recovery_journal *journal, struct vdo_completion *parent) { … } /** * vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the * VDO. * @journal: The journal. * * Return: The number of logical blocks in use by the VDO. */ block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal) { … } /** * vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal. * @journal: The recovery journal to query. * * Return: A copy of the current statistics for the journal. */ struct recovery_journal_statistics vdo_get_recovery_journal_statistics(const struct recovery_journal *journal) { … } /** * dump_recovery_block() - Dump the contents of the recovery block to the log. * @block: The block to dump. */ static void dump_recovery_block(const struct recovery_journal_block *block) { … } /** * vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from * the recovery journal. * @journal: The recovery journal to dump. */ void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal) { … }
Codebase Browser
linux