// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2023 Red Hat
*/
#include "recovery-journal.h"
#include <linux/atomic.h>
#include <linux/bio.h>
#include "logger.h"
#include "memory-alloc.h"
#include "permassert.h"
#include "block-map.h"
#include "completion.h"
#include "constants.h"
#include "data-vio.h"
#include "encodings.h"
#include "io-submitter.h"
#include "slab-depot.h"
#include "types.h"
#include "vdo.h"
#include "vio.h"
#include "wait-queue.h"
static const u64 RECOVERY_COUNT_MASK = 0xff;
/*
* The number of reserved blocks must be large enough to prevent a new recovery journal
* block write from overwriting a block which appears to still be a valid head block of the
* journal. Currently, that means reserving enough space for all 2048 data_vios.
*/
#define RECOVERY_JOURNAL_RESERVED_BLOCKS \
((MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2)
/**
* DOC: Lock Counters.
*
* A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The
* per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0
* is at index 0, zone 0's lock 1 is at index 1, and zone 1's lock 0 is at index 'locks'. This
* arrangement is intended to minimize cache-line contention for counters from different zones.
*
* The locks are implemented as a single object instead of as a lock counter per lock both to
* afford this opportunity to reduce cache line contention and also to eliminate the need to have a
* completion per lock.
*
* Lock sets are laid out with the set for recovery journal first, followed by the logical zones,
* and then the physical zones.
*/
enum lock_counter_state {
LOCK_COUNTER_STATE_NOT_NOTIFYING,
LOCK_COUNTER_STATE_NOTIFYING,
LOCK_COUNTER_STATE_SUSPENDED,
};
/**
* get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone.
* @journal: The recovery journal.
* @lock_number: The lock to get.
* @zone_type: The zone type whose count is desired.
*
* Return: A pointer to the zone count for the given lock and zone.
*/
static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal,
block_count_t lock_number,
enum vdo_zone_type zone_type)
{
return ((zone_type == VDO_ZONE_TYPE_LOGICAL)
? &journal->lock_counter.logical_zone_counts[lock_number]
: &journal->lock_counter.physical_zone_counts[lock_number]);
}
/**
* get_counter() - Get the zone counter for a given lock on a given zone.
* @journal: The recovery journal.
* @lock_number: The lock to get.
* @zone_type: The zone type whose count is desired.
* @zone_id: The zone index whose count is desired.
*
* Return: The counter for the given lock and zone.
*/
static inline u16 *get_counter(struct recovery_journal *journal,
block_count_t lock_number, enum vdo_zone_type zone_type,
zone_count_t zone_id)
{
struct lock_counter *counter = &journal->lock_counter;
block_count_t zone_counter = (counter->locks * zone_id) + lock_number;
if (zone_type == VDO_ZONE_TYPE_JOURNAL)
return &counter->journal_counters[zone_counter];
if (zone_type == VDO_ZONE_TYPE_LOGICAL)
return &counter->logical_counters[zone_counter];
return &counter->physical_counters[zone_counter];
}
static atomic_t *get_decrement_counter(struct recovery_journal *journal,
block_count_t lock_number)
{
return &journal->lock_counter.journal_decrement_counts[lock_number];
}
/**
* is_journal_zone_locked() - Check whether the journal zone is locked for a given lock.
* @journal: The recovery journal.
* @lock_number: The lock to check.
*
* Return: true if the journal zone is locked.
*/
static bool is_journal_zone_locked(struct recovery_journal *journal,
block_count_t lock_number)
{
u16 journal_value = *get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);
u32 decrements = atomic_read(get_decrement_counter(journal, lock_number));
/* Pairs with barrier in vdo_release_journal_entry_lock() */
smp_rmb();
VDO_ASSERT_LOG_ONLY((decrements <= journal_value),
"journal zone lock counter must not underflow");
return (journal_value != decrements);
}
/**
* vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal
* block.
* @journal: The recovery journal.
* @sequence_number: The journal sequence number of the referenced block.
* @zone_type: The type of the zone making the adjustment.
* @zone_id: The ID of the zone making the adjustment.
*
* If this is the last reference for a given zone type, an attempt will be made to reap the
* journal.
*/
void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal,
sequence_number_t sequence_number,
enum vdo_zone_type zone_type,
zone_count_t zone_id)
{
u16 *current_value;
block_count_t lock_number;
int prior_state;
if (sequence_number == 0)
return;
lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
current_value = get_counter(journal, lock_number, zone_type, zone_id);
VDO_ASSERT_LOG_ONLY((*current_value >= 1),
"decrement of lock counter must not underflow");
*current_value -= 1;
if (zone_type == VDO_ZONE_TYPE_JOURNAL) {
if (is_journal_zone_locked(journal, lock_number))
return;
} else {
atomic_t *zone_count;
if (*current_value != 0)
return;
zone_count = get_zone_count_ptr(journal, lock_number, zone_type);
if (atomic_add_return(-1, zone_count) > 0)
return;
}
/*
* Extra barriers because this was original developed using a CAS operation that implicitly
* had them.
*/
smp_mb__before_atomic();
prior_state = atomic_cmpxchg(&journal->lock_counter.state,
LOCK_COUNTER_STATE_NOT_NOTIFYING,
LOCK_COUNTER_STATE_NOTIFYING);
/* same as before_atomic */
smp_mb__after_atomic();
if (prior_state != LOCK_COUNTER_STATE_NOT_NOTIFYING)
return;
vdo_launch_completion(&journal->lock_counter.completion);
}
static inline struct recovery_journal_block * __must_check get_journal_block(struct list_head *list)
{
return list_first_entry_or_null(list, struct recovery_journal_block, list_node);
}
/**
* pop_free_list() - Get a block from the end of the free list.
* @journal: The journal.
*
* Return: The block or NULL if the list is empty.
*/
static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal)
{
struct recovery_journal_block *block;
if (list_empty(&journal->free_tail_blocks))
return NULL;
block = list_last_entry(&journal->free_tail_blocks,
struct recovery_journal_block, list_node);
list_del_init(&block->list_node);
return block;
}
/**
* is_block_dirty() - Check whether a recovery block is dirty.
* @block: The block to check.
*
* Indicates it has any uncommitted entries, which includes both entries not written and entries
* written but not yet acknowledged.
*
* Return: true if the block has any uncommitted entries.
*/
static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block)
{
return (block->uncommitted_entry_count > 0);
}
/**
* is_block_empty() - Check whether a journal block is empty.
* @block: The block to check.
*
* Return: true if the block has no entries.
*/
static inline bool __must_check is_block_empty(const struct recovery_journal_block *block)
{
return (block->entry_count == 0);
}
/**
* is_block_full() - Check whether a journal block is full.
* @block: The block to check.
*
* Return: true if the block is full.
*/
static inline bool __must_check is_block_full(const struct recovery_journal_block *block)
{
return ((block == NULL) || (block->journal->entries_per_block == block->entry_count));
}
/**
* assert_on_journal_thread() - Assert that we are running on the journal thread.
* @journal: The journal.
* @function_name: The function doing the check (for logging).
*/
static void assert_on_journal_thread(struct recovery_journal *journal,
const char *function_name)
{
VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id),
"%s() called on journal thread", function_name);
}
/**
* continue_waiter() - Release a data_vio from the journal.
*
* Invoked whenever a data_vio is to be released from the journal, either because its entry was
* committed to disk, or because there was an error. Implements waiter_callback_fn.
*/
static void continue_waiter(struct vdo_waiter *waiter, void *context)
{
continue_data_vio_with_error(vdo_waiter_as_data_vio(waiter), *((int *) context));
}
/**
* has_block_waiters() - Check whether the journal has any waiters on any blocks.
* @journal: The journal in question.
*
* Return: true if any block has a waiter.
*/
static inline bool has_block_waiters(struct recovery_journal *journal)
{
struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks);
/*
* Either the first active tail block (if it exists) has waiters, or no active tail block
* has waiters.
*/
return ((block != NULL) &&
(vdo_waitq_has_waiters(&block->entry_waiters) ||
vdo_waitq_has_waiters(&block->commit_waiters)));
}
static void recycle_journal_blocks(struct recovery_journal *journal);
static void recycle_journal_block(struct recovery_journal_block *block);
static void notify_commit_waiters(struct recovery_journal *journal);
/**
* suspend_lock_counter() - Prevent the lock counter from notifying.
* @counter: The counter.
*
* Return: true if the lock counter was not notifying and hence the suspend was efficacious.
*/
static bool suspend_lock_counter(struct lock_counter *counter)
{
int prior_state;
/*
* Extra barriers because this was originally developed using a CAS operation that
* implicitly had them.
*/
smp_mb__before_atomic();
prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_NOT_NOTIFYING,
LOCK_COUNTER_STATE_SUSPENDED);
/* same as before_atomic */
smp_mb__after_atomic();
return ((prior_state == LOCK_COUNTER_STATE_SUSPENDED) ||
(prior_state == LOCK_COUNTER_STATE_NOT_NOTIFYING));
}
static inline bool is_read_only(struct recovery_journal *journal)
{
return vdo_is_read_only(journal->flush_vio->completion.vdo);
}
/**
* check_for_drain_complete() - Check whether the journal has drained.
* @journal: The journal which may have just drained.
*/
static void check_for_drain_complete(struct recovery_journal *journal)
{
int result = VDO_SUCCESS;
if (is_read_only(journal)) {
result = VDO_READ_ONLY;
/*
* Clean up any full active blocks which were not written due to read-only mode.
*
* FIXME: This would probably be better as a short-circuit in write_block().
*/
notify_commit_waiters(journal);
recycle_journal_blocks(journal);
/* Release any data_vios waiting to be assigned entries. */
vdo_waitq_notify_all_waiters(&journal->entry_waiters,
continue_waiter, &result);
}
if (!vdo_is_state_draining(&journal->state) ||
journal->reaping ||
has_block_waiters(journal) ||
vdo_waitq_has_waiters(&journal->entry_waiters) ||
!suspend_lock_counter(&journal->lock_counter))
return;
if (vdo_is_state_saving(&journal->state)) {
if (journal->active_block != NULL) {
VDO_ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) ||
!is_block_dirty(journal->active_block)),
"journal being saved has clean active block");
recycle_journal_block(journal->active_block);
}
VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),
"all blocks in a journal being saved must be inactive");
}
vdo_finish_draining_with_result(&journal->state, result);
}
/**
* notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone
* read-only.
* @listener: The journal.
* @parent: The completion to notify in order to acknowledge the notification.
*
* Implements vdo_read_only_notification_fn.
*/
static void notify_recovery_journal_of_read_only_mode(void *listener,
struct vdo_completion *parent)
{
check_for_drain_complete(listener);
vdo_finish_completion(parent);
}
/**
* enter_journal_read_only_mode() - Put the journal in read-only mode.
* @journal: The journal which has failed.
* @error_code: The error result triggering this call.
*
* All attempts to add entries after this function is called will fail. All VIOs waiting for
* commits will be awakened with an error.
*/
static void enter_journal_read_only_mode(struct recovery_journal *journal,
int error_code)
{
vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code);
check_for_drain_complete(journal);
}
/**
* vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current
* sequence number.
* @journal: The journal in question.
*
* Exposed only so the block map can be initialized therefrom.
*
* Return: The sequence number of the tail block.
*/
sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal)
{
return journal->tail;
}
/**
* get_recovery_journal_head() - Get the head of the recovery journal.
* @journal: The journal.
*
* The head is the lowest sequence number of the block map head and the slab journal head.
*
* Return: the head of the journal.
*/
static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal)
{
return min(journal->block_map_head, journal->slab_journal_head);
}
/**
* compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count.
* @recovery_count: The recovery count.
*
* Return: The byte corresponding to the recovery count.
*/
static inline u8 __must_check compute_recovery_count_byte(u64 recovery_count)
{
return (u8)(recovery_count & RECOVERY_COUNT_MASK);
}
/**
* check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if
* so, force the oldest slab journal tail block to commit.
* @journal: The journal.
*/
static void check_slab_journal_commit_threshold(struct recovery_journal *journal)
{
block_count_t current_length = journal->tail - journal->slab_journal_head;
if (current_length > journal->slab_journal_commit_threshold) {
journal->events.slab_journal_commits_requested++;
vdo_commit_oldest_slab_journal_tail_blocks(journal->depot,
journal->slab_journal_head);
}
}
static void reap_recovery_journal(struct recovery_journal *journal);
static void assign_entries(struct recovery_journal *journal);
/**
* finish_reaping() - Finish reaping the journal.
* @journal: The journal being reaped.
*/
static void finish_reaping(struct recovery_journal *journal)
{
block_count_t blocks_reaped;
sequence_number_t old_head = get_recovery_journal_head(journal);
journal->block_map_head = journal->block_map_reap_head;
journal->slab_journal_head = journal->slab_journal_reap_head;
blocks_reaped = get_recovery_journal_head(journal) - old_head;
journal->available_space += blocks_reaped * journal->entries_per_block;
journal->reaping = false;
check_slab_journal_commit_threshold(journal);
assign_entries(journal);
check_for_drain_complete(journal);
}
/**
* complete_reaping() - Finish reaping the journal after flushing the lower layer.
* @completion: The journal's flush VIO.
*
* This is the callback registered in reap_recovery_journal().
*/
static void complete_reaping(struct vdo_completion *completion)
{
struct recovery_journal *journal = completion->parent;
finish_reaping(journal);
/* Try reaping again in case more locks were released while flush was out. */
reap_recovery_journal(journal);
}
/**
* handle_flush_error() - Handle an error when flushing the lower layer due to reaping.
* @completion: The journal's flush VIO.
*/
static void handle_flush_error(struct vdo_completion *completion)
{
struct recovery_journal *journal = completion->parent;
vio_record_metadata_io_error(as_vio(completion));
journal->reaping = false;
enter_journal_read_only_mode(journal, completion->result);
}
static void flush_endio(struct bio *bio)
{
struct vio *vio = bio->bi_private;
struct recovery_journal *journal = vio->completion.parent;
continue_vio_after_io(vio, complete_reaping, journal->thread_id);
}
/**
* initialize_journal_state() - Set all journal fields appropriately to start journaling from the
* current active block.
* @journal: The journal to be reset based on its active block.
*/
static void initialize_journal_state(struct recovery_journal *journal)
{
journal->append_point.sequence_number = journal->tail;
journal->last_write_acknowledged = journal->tail;
journal->block_map_head = journal->tail;
journal->slab_journal_head = journal->tail;
journal->block_map_reap_head = journal->tail;
journal->slab_journal_reap_head = journal->tail;
journal->block_map_head_block_number =
vdo_get_recovery_journal_block_number(journal, journal->block_map_head);
journal->slab_journal_head_block_number =
vdo_get_recovery_journal_block_number(journal,
journal->slab_journal_head);
journal->available_space =
(journal->entries_per_block * vdo_get_recovery_journal_length(journal->size));
}
/**
* vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks.
* @journal_size: The size of the recovery journal in blocks.
*
* Return: the number of recovery journal blocks usable for entries.
*/
block_count_t vdo_get_recovery_journal_length(block_count_t journal_size)
{
block_count_t reserved_blocks = journal_size / 4;
if (reserved_blocks > RECOVERY_JOURNAL_RESERVED_BLOCKS)
reserved_blocks = RECOVERY_JOURNAL_RESERVED_BLOCKS;
return (journal_size - reserved_blocks);
}
/**
* reap_recovery_journal_callback() - Attempt to reap the journal.
* @completion: The lock counter completion.
*
* Attempts to reap the journal now that all the locks on some journal block have been released.
* This is the callback registered with the lock counter.
*/
static void reap_recovery_journal_callback(struct vdo_completion *completion)
{
struct recovery_journal *journal = (struct recovery_journal *) completion->parent;
/*
* The acknowledgment must be done before reaping so that there is no race between
* acknowledging the notification and unlocks wishing to notify.
*/
smp_wmb();
atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING);
if (vdo_is_state_quiescing(&journal->state)) {
/*
* Don't start reaping when the journal is trying to quiesce. Do check if this
* notification is the last thing the is waiting on.
*/
check_for_drain_complete(journal);
return;
}
reap_recovery_journal(journal);
check_slab_journal_commit_threshold(journal);
}
/**
* initialize_lock_counter() - Initialize a lock counter.
*
* @journal: The recovery journal.
* @vdo: The vdo.
*
* Return: VDO_SUCCESS or an error.
*/
static int __must_check initialize_lock_counter(struct recovery_journal *journal,
struct vdo *vdo)
{
int result;
struct thread_config *config = &vdo->thread_config;
struct lock_counter *counter = &journal->lock_counter;
result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters);
if (result != VDO_SUCCESS)
return result;
result = vdo_allocate(journal->size, atomic_t, __func__,
&counter->journal_decrement_counts);
if (result != VDO_SUCCESS)
return result;
result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__,
&counter->logical_counters);
if (result != VDO_SUCCESS)
return result;
result = vdo_allocate(journal->size, atomic_t, __func__,
&counter->logical_zone_counts);
if (result != VDO_SUCCESS)
return result;
result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__,
&counter->physical_counters);
if (result != VDO_SUCCESS)
return result;
result = vdo_allocate(journal->size, atomic_t, __func__,
&counter->physical_zone_counts);
if (result != VDO_SUCCESS)
return result;
vdo_initialize_completion(&counter->completion, vdo,
VDO_LOCK_COUNTER_COMPLETION);
vdo_prepare_completion(&counter->completion, reap_recovery_journal_callback,
reap_recovery_journal_callback, config->journal_thread,
journal);
counter->logical_zones = config->logical_zone_count;
counter->physical_zones = config->physical_zone_count;
counter->locks = journal->size;
return VDO_SUCCESS;
}
/**
* set_journal_tail() - Set the journal's tail sequence number.
* @journal: The journal whose tail is to be set.
* @tail: The new tail value.
*/
static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail)
{
/* VDO does not support sequence numbers above 1 << 48 in the slab journal. */
if (tail >= (1ULL << 48))
enter_journal_read_only_mode(journal, VDO_JOURNAL_OVERFLOW);
journal->tail = tail;
}
/**
* initialize_recovery_block() - Initialize a journal block.
* @vdo: The vdo from which to construct vios.
* @journal: The journal to which the block will belong.
* @block: The block to initialize.
*
* Return: VDO_SUCCESS or an error.
*/
static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal,
struct recovery_journal_block *block)
{
char *data;
int result;
/*
* Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries.
*/
BUILD_BUG_ON(RECOVERY_JOURNAL_ENTRIES_PER_BLOCK >
((VDO_BLOCK_SIZE - sizeof(struct packed_journal_header)) /
sizeof(struct packed_recovery_journal_entry)));
/*
* Allocate a full block for the journal block even though not all of the space is used
* since the VIO needs to write a full disk block.
*/
result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &data);
if (result != VDO_SUCCESS)
return result;
result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL,
VIO_PRIORITY_HIGH, block, 1, data, &block->vio);
if (result != VDO_SUCCESS) {
vdo_free(data);
return result;
}
list_add_tail(&block->list_node, &journal->free_tail_blocks);
block->journal = journal;
return VDO_SUCCESS;
}
/**
* vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that
* was decoded from the super block.
*
* @state: The decoded state of the journal.
* @nonce: The nonce of the VDO.
* @vdo: The VDO.
* @partition: The partition for the journal.
* @recovery_count: The VDO's number of completed recoveries.
* @journal_size: The number of blocks in the journal on disk.
* @journal_ptr: The pointer to hold the new recovery journal.
*
* Return: A success or error code.
*/
int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t nonce,
struct vdo *vdo, struct partition *partition,
u64 recovery_count, block_count_t journal_size,
struct recovery_journal **journal_ptr)
{
block_count_t i;
struct recovery_journal *journal;
int result;
result = vdo_allocate_extended(struct recovery_journal,
RECOVERY_JOURNAL_RESERVED_BLOCKS,
struct recovery_journal_block, __func__,
&journal);
if (result != VDO_SUCCESS)
return result;
INIT_LIST_HEAD(&journal->free_tail_blocks);
INIT_LIST_HEAD(&journal->active_tail_blocks);
vdo_waitq_init(&journal->pending_writes);
journal->thread_id = vdo->thread_config.journal_thread;
journal->origin = partition->offset;
journal->nonce = nonce;
journal->recovery_count = compute_recovery_count_byte(recovery_count);
journal->size = journal_size;
journal->slab_journal_commit_threshold = (journal_size * 2) / 3;
journal->logical_blocks_used = state.logical_blocks_used;
journal->block_map_data_blocks = state.block_map_data_blocks;
journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK;
set_journal_tail(journal, state.journal_start);
initialize_journal_state(journal);
/* TODO: this will have to change if we make initial resume of a VDO a real resume */
vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED);
for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) {
struct recovery_journal_block *block = &journal->blocks[i];
result = initialize_recovery_block(vdo, journal, block);
if (result != VDO_SUCCESS) {
vdo_free_recovery_journal(journal);
return result;
}
}
result = initialize_lock_counter(journal, vdo);
if (result != VDO_SUCCESS) {
vdo_free_recovery_journal(journal);
return result;
}
result = create_metadata_vio(vdo, VIO_TYPE_RECOVERY_JOURNAL, VIO_PRIORITY_HIGH,
journal, NULL, &journal->flush_vio);
if (result != VDO_SUCCESS) {
vdo_free_recovery_journal(journal);
return result;
}
result = vdo_register_read_only_listener(vdo, journal,
notify_recovery_journal_of_read_only_mode,
journal->thread_id);
if (result != VDO_SUCCESS) {
vdo_free_recovery_journal(journal);
return result;
}
result = vdo_make_default_thread(vdo, journal->thread_id);
if (result != VDO_SUCCESS) {
vdo_free_recovery_journal(journal);
return result;
}
journal->flush_vio->completion.callback_thread_id = journal->thread_id;
*journal_ptr = journal;
return VDO_SUCCESS;
}
/**
* vdo_free_recovery_journal() - Free a recovery journal.
* @journal: The recovery journal to free.
*/
void vdo_free_recovery_journal(struct recovery_journal *journal)
{
block_count_t i;
if (journal == NULL)
return;
vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts));
vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts));
vdo_free(vdo_forget(journal->lock_counter.journal_counters));
vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts));
vdo_free(vdo_forget(journal->lock_counter.logical_counters));
vdo_free(vdo_forget(journal->lock_counter.physical_counters));
free_vio(vdo_forget(journal->flush_vio));
/*
* FIXME: eventually, the journal should be constructed in a quiescent state which
* requires opening before use.
*/
if (!vdo_is_state_quiescent(&journal->state)) {
VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),
"journal being freed has no active tail blocks");
} else if (!vdo_is_state_saved(&journal->state) &&
!list_empty(&journal->active_tail_blocks)) {
vdo_log_warning("journal being freed has uncommitted entries");
}
for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) {
struct recovery_journal_block *block = &journal->blocks[i];
vdo_free(vdo_forget(block->vio.data));
free_vio_components(&block->vio);
}
vdo_free(journal);
}
/**
* vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair.
* @journal: The journal in question.
* @recovery_count: The number of completed recoveries.
* @tail: The new tail block sequence number.
* @logical_blocks_used: The new number of logical blocks used.
* @block_map_data_blocks: The new number of block map data blocks.
*/
void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal,
u64 recovery_count,
sequence_number_t tail,
block_count_t logical_blocks_used,
block_count_t block_map_data_blocks)
{
set_journal_tail(journal, tail + 1);
journal->recovery_count = compute_recovery_count_byte(recovery_count);
initialize_journal_state(journal);
journal->logical_blocks_used = logical_blocks_used;
journal->block_map_data_blocks = block_map_data_blocks;
}
/**
* vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from
* data blocks, currently in use.
* @journal: The journal in question.
*
* Return: The number of block map pages allocated from slabs.
*/
block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal)
{
return journal->block_map_data_blocks;
}
/**
* vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread.
* @journal: The journal to query.
*
* Return: The ID of the journal's thread.
*/
thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal)
{
return journal->thread_id;
}
/**
* vdo_open_recovery_journal() - Prepare the journal for new entries.
* @journal: The journal in question.
* @depot: The slab depot for this VDO.
* @block_map: The block map for this VDO.
*/
void vdo_open_recovery_journal(struct recovery_journal *journal,
struct slab_depot *depot, struct block_map *block_map)
{
journal->depot = depot;
journal->block_map = block_map;
WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION);
}
/**
* vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super
* block.
* @journal: the recovery journal.
*
* Return: the state of the journal.
*/
struct recovery_journal_state_7_0
vdo_record_recovery_journal(const struct recovery_journal *journal)
{
struct recovery_journal_state_7_0 state = {
.logical_blocks_used = journal->logical_blocks_used,
.block_map_data_blocks = journal->block_map_data_blocks,
};
if (vdo_is_state_saved(&journal->state)) {
/*
* If the journal is saved, we should start one past the active block (since the
* active block is not guaranteed to be empty).
*/
state.journal_start = journal->tail;
} else {
/*
* When we're merely suspended or have gone read-only, we must record the first
* block that might have entries that need to be applied.
*/
state.journal_start = get_recovery_journal_head(journal);
}
return state;
}
/**
* get_block_header() - Get a pointer to the packed journal block header in the block buffer.
* @block: The recovery block.
*
* Return: The block's header.
*/
static inline struct packed_journal_header *
get_block_header(const struct recovery_journal_block *block)
{
return (struct packed_journal_header *) block->vio.data;
}
/**
* set_active_sector() - Set the current sector of the current block and initialize it.
* @block: The block to update.
* @sector: A pointer to the first byte of the new sector.
*/
static void set_active_sector(struct recovery_journal_block *block, void *sector)
{
block->sector = sector;
block->sector->check_byte = get_block_header(block)->check_byte;
block->sector->recovery_count = block->journal->recovery_count;
block->sector->entry_count = 0;
}
/**
* advance_tail() - Advance the tail of the journal.
* @journal: The journal whose tail should be advanced.
*
* Return: true if the tail was advanced.
*/
static bool advance_tail(struct recovery_journal *journal)
{
struct recovery_block_header unpacked;
struct packed_journal_header *header;
struct recovery_journal_block *block;
block = journal->active_block = pop_free_list(journal);
if (block == NULL)
return false;
list_move_tail(&block->list_node, &journal->active_tail_blocks);
unpacked = (struct recovery_block_header) {
.metadata_type = VDO_METADATA_RECOVERY_JOURNAL_2,
.block_map_data_blocks = journal->block_map_data_blocks,
.logical_blocks_used = journal->logical_blocks_used,
.nonce = journal->nonce,
.recovery_count = journal->recovery_count,
.sequence_number = journal->tail,
.check_byte = vdo_compute_recovery_journal_check_byte(journal,
journal->tail),
};
header = get_block_header(block);
memset(block->vio.data, 0x0, VDO_BLOCK_SIZE);
block->sequence_number = journal->tail;
block->entry_count = 0;
block->uncommitted_entry_count = 0;
block->block_number = vdo_get_recovery_journal_block_number(journal,
journal->tail);
vdo_pack_recovery_block_header(&unpacked, header);
set_active_sector(block, vdo_get_journal_block_sector(header, 1));
set_journal_tail(journal, journal->tail + 1);
vdo_advance_block_map_era(journal->block_map, journal->tail);
return true;
}
/**
* initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock.
* @journal: The recovery journal.
*
* Context: This must be called from the journal zone.
*/
static void initialize_lock_count(struct recovery_journal *journal)
{
u16 *journal_value;
block_count_t lock_number = journal->active_block->block_number;
atomic_t *decrement_counter = get_decrement_counter(journal, lock_number);
journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);
VDO_ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)),
"count to be initialized not in use");
*journal_value = journal->entries_per_block + 1;
atomic_set(decrement_counter, 0);
}
/**
* prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check
* whether an entry of the given type may be assigned at this time.
* @journal: The journal receiving an entry.
*
* Return: true if there is space in the journal to store an entry of the specified type.
*/
static bool prepare_to_assign_entry(struct recovery_journal *journal)
{
if (journal->available_space == 0)
return false;
if (is_block_full(journal->active_block) && !advance_tail(journal))
return false;
if (!is_block_empty(journal->active_block))
return true;
if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) {
/* Cannot use this block since the journal is full. */
journal->events.disk_full++;
return false;
}
/*
* Don't allow the new block to be reaped until all of its entries have been committed to
* the block map and until the journal block has been fully committed as well. Because the
* block map update is done only after any slab journal entries have been made, the
* per-entry lock for the block map entry serves to protect those as well.
*/
initialize_lock_count(journal);
return true;
}
static void write_blocks(struct recovery_journal *journal);
/**
* schedule_block_write() - Queue a block for writing.
* @journal: The journal in question.
* @block: The block which is now ready to write.
*
* The block is expected to be full. If the block is currently writing, this is a noop as the block
* will be queued for writing when the write finishes. The block must not currently be queued for
* writing.
*/
static void schedule_block_write(struct recovery_journal *journal,
struct recovery_journal_block *block)
{
if (!block->committing)
vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter);
/*
* At the end of adding entries, or discovering this partial block is now full and ready to
* rewrite, we will call write_blocks() and write a whole batch.
*/
}
/**
* release_journal_block_reference() - Release a reference to a journal block.
* @block: The journal block from which to release a reference.
*/
static void release_journal_block_reference(struct recovery_journal_block *block)
{
vdo_release_recovery_journal_block_reference(block->journal,
block->sequence_number,
VDO_ZONE_TYPE_JOURNAL, 0);
}
static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio)
{
if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) {
journal->block_map_data_blocks++;
return;
}
if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED)
journal->logical_blocks_used++;
if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED)
journal->logical_blocks_used--;
}
/**
* assign_entry() - Assign an entry waiter to the active block.
*
* Implements waiter_callback_fn.
*/
static void assign_entry(struct vdo_waiter *waiter, void *context)
{
struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);
struct recovery_journal_block *block = context;
struct recovery_journal *journal = block->journal;
/* Record the point at which we will make the journal entry. */
data_vio->recovery_journal_point = (struct journal_point) {
.sequence_number = block->sequence_number,
.entry_count = block->entry_count,
};
update_usages(journal, data_vio);
journal->available_space--;
if (!vdo_waitq_has_waiters(&block->entry_waiters))
journal->events.blocks.started++;
vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);
block->entry_count++;
block->uncommitted_entry_count++;
journal->events.entries.started++;
if (is_block_full(block)) {
/*
* The block is full, so we can write it anytime henceforth. If it is already
* committing, we'll queue it for writing when it comes back.
*/
schedule_block_write(journal, block);
}
/* Force out slab journal tail blocks when threshold is reached. */
check_slab_journal_commit_threshold(journal);
}
static void assign_entries(struct recovery_journal *journal)
{
if (journal->adding_entries) {
/* Protect against re-entrancy. */
return;
}
journal->adding_entries = true;
while (vdo_waitq_has_waiters(&journal->entry_waiters) &&
prepare_to_assign_entry(journal)) {
vdo_waitq_notify_next_waiter(&journal->entry_waiters,
assign_entry, journal->active_block);
}
/* Now that we've finished with entries, see if we have a batch of blocks to write. */
write_blocks(journal);
journal->adding_entries = false;
}
/**
* recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been
* fully committed.
* @block: The block to be recycled.
*/
static void recycle_journal_block(struct recovery_journal_block *block)
{
struct recovery_journal *journal = block->journal;
block_count_t i;
list_move_tail(&block->list_node, &journal->free_tail_blocks);
/* Release any unused entry locks. */
for (i = block->entry_count; i < journal->entries_per_block; i++)
release_journal_block_reference(block);
/*
* Release our own lock against reaping now that the block is completely committed, or
* we're giving up because we're in read-only mode.
*/
if (block->entry_count > 0)
release_journal_block_reference(block);
if (block == journal->active_block)
journal->active_block = NULL;
}
/**
* continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because
* its entry was committed to disk.
*
* Implements waiter_callback_fn.
*/
static void continue_committed_waiter(struct vdo_waiter *waiter, void *context)
{
struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);
struct recovery_journal *journal = context;
int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS);
bool has_decrement;
VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point,
&data_vio->recovery_journal_point),
"DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)",
(unsigned long long) journal->commit_point.sequence_number,
journal->commit_point.entry_count,
(unsigned long long) data_vio->recovery_journal_point.sequence_number,
data_vio->recovery_journal_point.entry_count);
journal->commit_point = data_vio->recovery_journal_point;
data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS;
if (result != VDO_SUCCESS) {
continue_data_vio_with_error(data_vio, result);
return;
}
/*
* The increment must be launched first since it must come before the
* decrement if they are in the same slab.
*/
has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK);
if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement)
continue_data_vio(data_vio);
if (has_decrement)
vdo_launch_completion(&data_vio->decrement_completion);
}
/**
* notify_commit_waiters() - Notify any VIOs whose entries have now committed.
* @journal: The recovery journal to update.
*/
static void notify_commit_waiters(struct recovery_journal *journal)
{
struct recovery_journal_block *block;
list_for_each_entry(block, &journal->active_tail_blocks, list_node) {
if (block->committing)
return;
vdo_waitq_notify_all_waiters(&block->commit_waiters,
continue_committed_waiter, journal);
if (is_read_only(journal)) {
vdo_waitq_notify_all_waiters(&block->entry_waiters,
continue_committed_waiter,
journal);
} else if (is_block_dirty(block) || !is_block_full(block)) {
/* Stop at partially-committed or partially-filled blocks. */
return;
}
}
}
/**
* recycle_journal_blocks() - Recycle any journal blocks which have been fully committed.
* @journal: The recovery journal to update.
*/
static void recycle_journal_blocks(struct recovery_journal *journal)
{
struct recovery_journal_block *block, *tmp;
list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) {
if (block->committing) {
/* Don't recycle committing blocks. */
return;
}
if (!is_read_only(journal) &&
(is_block_dirty(block) || !is_block_full(block))) {
/*
* Don't recycle partially written or partially full blocks, except in
* read-only mode.
*/
return;
}
recycle_journal_block(block);
}
}
/**
* complete_write() - Handle post-commit processing.
* @completion: The completion of the VIO writing this block.
*
* This is the callback registered by write_block(). If more entries accumulated in the block being
* committed while the commit was in progress, another commit will be initiated.
*/
static void complete_write(struct vdo_completion *completion)
{
struct recovery_journal_block *block = completion->parent;
struct recovery_journal *journal = block->journal;
struct recovery_journal_block *last_active_block;
assert_on_journal_thread(journal, __func__);
journal->pending_write_count -= 1;
journal->events.blocks.committed += 1;
journal->events.entries.committed += block->entries_in_commit;
block->uncommitted_entry_count -= block->entries_in_commit;
block->entries_in_commit = 0;
block->committing = false;
/* If this block is the latest block to be acknowledged, record that fact. */
if (block->sequence_number > journal->last_write_acknowledged)
journal->last_write_acknowledged = block->sequence_number;
last_active_block = get_journal_block(&journal->active_tail_blocks);
VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number),
"completed journal write is still active");
notify_commit_waiters(journal);
/*
* Is this block now full? Reaping, and adding entries, might have already sent it off for
* rewriting; else, queue it for rewrite.
*/
if (is_block_dirty(block) && is_block_full(block))
schedule_block_write(journal, block);
recycle_journal_blocks(journal);
write_blocks(journal);
check_for_drain_complete(journal);
}
static void handle_write_error(struct vdo_completion *completion)
{
struct recovery_journal_block *block = completion->parent;
struct recovery_journal *journal = block->journal;
vio_record_metadata_io_error(as_vio(completion));
vdo_log_error_strerror(completion->result,
"cannot write recovery journal block %llu",
(unsigned long long) block->sequence_number);
enter_journal_read_only_mode(journal, completion->result);
complete_write(completion);
}
static void complete_write_endio(struct bio *bio)
{
struct vio *vio = bio->bi_private;
struct recovery_journal_block *block = vio->completion.parent;
struct recovery_journal *journal = block->journal;
continue_vio_after_io(vio, complete_write, journal->thread_id);
}
/**
* add_queued_recovery_entries() - Actually add entries from the queue to the given block.
* @block: The journal block.
*/
static void add_queued_recovery_entries(struct recovery_journal_block *block)
{
while (vdo_waitq_has_waiters(&block->entry_waiters)) {
struct data_vio *data_vio =
vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters));
struct tree_lock *lock = &data_vio->tree_lock;
struct packed_recovery_journal_entry *packed_entry;
struct recovery_journal_entry new_entry;
if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR)
set_active_sector(block,
(char *) block->sector + VDO_SECTOR_SIZE);
/* Compose and encode the entry. */
packed_entry = &block->sector->entries[block->sector->entry_count++];
new_entry = (struct recovery_journal_entry) {
.mapping = {
.pbn = data_vio->increment_updater.zpbn.pbn,
.state = data_vio->increment_updater.zpbn.state,
},
.unmapping = {
.pbn = data_vio->decrement_updater.zpbn.pbn,
.state = data_vio->decrement_updater.zpbn.state,
},
.operation = data_vio->increment_updater.operation,
.slot = lock->tree_slots[lock->height].block_map_slot,
};
*packed_entry = vdo_pack_recovery_journal_entry(&new_entry);
data_vio->recovery_sequence_number = block->sequence_number;
/* Enqueue the data_vio to wait for its entry to commit. */
vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);
}
}
/**
* write_block() - Issue a block for writing.
*
* Implements waiter_callback_fn.
*/
static void write_block(struct vdo_waiter *waiter, void *context __always_unused)
{
struct recovery_journal_block *block =
container_of(waiter, struct recovery_journal_block, write_waiter);
struct recovery_journal *journal = block->journal;
struct packed_journal_header *header = get_block_header(block);
if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) ||
is_read_only(journal))
return;
block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters);
add_queued_recovery_entries(block);
journal->pending_write_count += 1;
journal->events.blocks.written += 1;
journal->events.entries.written += block->entries_in_commit;
header->block_map_head = __cpu_to_le64(journal->block_map_head);
header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head);
header->entry_count = __cpu_to_le16(block->entry_count);
block->committing = true;
/*
* We must issue a flush and a FUA for every commit. The flush is necessary to ensure that
* the data being referenced is stable. The FUA is necessary to ensure that the journal
* block itself is stable before allowing overwrites of the lbn's previous data.
*/
vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number,
complete_write_endio, handle_write_error,
REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA);
}
/**
* write_blocks() - Attempt to commit blocks, according to write policy.
* @journal: The recovery journal.
*/
static void write_blocks(struct recovery_journal *journal)
{
assert_on_journal_thread(journal, __func__);
/*
* We call this function after adding entries to the journal and after finishing a block
* write. Thus, when this function terminates we must either have no VIOs waiting in the
* journal or have some outstanding IO to provide a future wakeup.
*
* We want to only issue full blocks if there are no pending writes. However, if there are
* no outstanding writes and some unwritten entries, we must issue a block, even if it's
* the active block and it isn't full.
*/
if (journal->pending_write_count > 0)
return;
/* Write all the full blocks. */
vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL);
/*
* Do we need to write the active block? Only if we have no outstanding writes, even after
* issuing all of the full writes.
*/
if ((journal->pending_write_count == 0) && (journal->active_block != NULL))
write_block(&journal->active_block->write_waiter, NULL);
}
/**
* vdo_add_recovery_journal_entry() - Add an entry to a recovery journal.
* @journal: The journal in which to make an entry.
* @data_vio: The data_vio for which to add the entry. The entry will be taken
* from the logical and new_mapped fields of the data_vio. The
* data_vio's recovery_sequence_number field will be set to the
* sequence number of the journal block in which the entry was
* made.
*
* This method is asynchronous. The data_vio will not be called back until the entry is committed
* to the on-disk journal.
*/
void vdo_add_recovery_journal_entry(struct recovery_journal *journal,
struct data_vio *data_vio)
{
assert_on_journal_thread(journal, __func__);
if (!vdo_is_state_normal(&journal->state)) {
continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE);
return;
}
if (is_read_only(journal)) {
continue_data_vio_with_error(data_vio, VDO_READ_ONLY);
return;
}
VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0,
"journal lock not held for new entry");
vdo_advance_journal_point(&journal->append_point, journal->entries_per_block);
vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter);
assign_entries(journal);
}
/**
* is_lock_locked() - Check whether a lock is locked for a zone type.
* @journal: The recovery journal.
* @lock_number: The lock to check.
* @zone_type: The type of the zone.
*
* If the recovery journal has a lock on the lock number, both logical and physical zones are
* considered locked.
*
* Return: true if the specified lock has references (is locked).
*/
static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number,
enum vdo_zone_type zone_type)
{
atomic_t *zone_count;
bool locked;
if (is_journal_zone_locked(journal, lock_number))
return true;
zone_count = get_zone_count_ptr(journal, lock_number, zone_type);
locked = (atomic_read(zone_count) != 0);
/* Pairs with implicit barrier in vdo_release_recovery_journal_block_reference() */
smp_rmb();
return locked;
}
/**
* reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks.
* @journal: The recovery journal.
*/
static void reap_recovery_journal(struct recovery_journal *journal)
{
if (journal->reaping) {
/*
* We already have an outstanding reap in progress. We need to wait for it to
* finish.
*/
return;
}
if (vdo_is_state_quiescent(&journal->state)) {
/* We are supposed to not do IO. Don't botch it by reaping. */
return;
}
/*
* Start reclaiming blocks only when the journal head has no references. Then stop when a
* block is referenced.
*/
while ((journal->block_map_reap_head < journal->last_write_acknowledged) &&
!is_lock_locked(journal, journal->block_map_head_block_number,
VDO_ZONE_TYPE_LOGICAL)) {
journal->block_map_reap_head++;
if (++journal->block_map_head_block_number == journal->size)
journal->block_map_head_block_number = 0;
}
while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) &&
!is_lock_locked(journal, journal->slab_journal_head_block_number,
VDO_ZONE_TYPE_PHYSICAL)) {
journal->slab_journal_reap_head++;
if (++journal->slab_journal_head_block_number == journal->size)
journal->slab_journal_head_block_number = 0;
}
if ((journal->block_map_reap_head == journal->block_map_head) &&
(journal->slab_journal_reap_head == journal->slab_journal_head)) {
/* Nothing happened. */
return;
}
/*
* If the block map head will advance, we must flush any block map page modified by the
* entries we are reaping. If the slab journal head will advance, we must flush the slab
* summary update covering the slab journal that just released some lock.
*/
journal->reaping = true;
vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error);
}
/**
* vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block
* from somewhere other than the journal itself.
* @journal: The recovery journal.
* @sequence_number: The journal sequence number of the referenced block.
* @zone_type: The type of the zone making the adjustment.
* @zone_id: The ID of the zone making the adjustment.
*/
void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal,
sequence_number_t sequence_number,
enum vdo_zone_type zone_type,
zone_count_t zone_id)
{
block_count_t lock_number;
u16 *current_value;
if (sequence_number == 0)
return;
VDO_ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL),
"invalid lock count increment from journal zone");
lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
current_value = get_counter(journal, lock_number, zone_type, zone_id);
VDO_ASSERT_LOG_ONLY(*current_value < U16_MAX,
"increment of lock counter must not overflow");
if (*current_value == 0) {
/*
* This zone is acquiring this lock for the first time. Extra barriers because this
* was original developed using an atomic add operation that implicitly had them.
*/
smp_mb__before_atomic();
atomic_inc(get_zone_count_ptr(journal, lock_number, zone_type));
/* same as before_atomic */
smp_mb__after_atomic();
}
*current_value += 1;
}
/**
* vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery
* journal block.
* @journal: The recovery journal.
* @sequence_number: The journal sequence number of the referenced block.
*/
void vdo_release_journal_entry_lock(struct recovery_journal *journal,
sequence_number_t sequence_number)
{
block_count_t lock_number;
if (sequence_number == 0)
return;
lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
/*
* Extra barriers because this was originally developed using an atomic add operation that
* implicitly had them.
*/
smp_mb__before_atomic();
atomic_inc(get_decrement_counter(journal, lock_number));
/* same as before_atomic */
smp_mb__after_atomic();
}
/**
* initiate_drain() - Initiate a drain.
*
* Implements vdo_admin_initiator_fn.
*/
static void initiate_drain(struct admin_state *state)
{
check_for_drain_complete(container_of(state, struct recovery_journal, state));
}
/**
* vdo_drain_recovery_journal() - Drain recovery journal I/O.
* @journal: The journal to drain.
* @operation: The drain operation (suspend or save).
* @parent: The completion to notify once the journal is drained.
*
* All uncommitted entries will be written out.
*/
void vdo_drain_recovery_journal(struct recovery_journal *journal,
const struct admin_state_code *operation,
struct vdo_completion *parent)
{
assert_on_journal_thread(journal, __func__);
vdo_start_draining(&journal->state, operation, parent, initiate_drain);
}
/**
* resume_lock_counter() - Re-allow notifications from a suspended lock counter.
* @counter: The counter.
*
* Return: true if the lock counter was suspended.
*/
static bool resume_lock_counter(struct lock_counter *counter)
{
int prior_state;
/*
* Extra barriers because this was original developed using a CAS operation that implicitly
* had them.
*/
smp_mb__before_atomic();
prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_SUSPENDED,
LOCK_COUNTER_STATE_NOT_NOTIFYING);
/* same as before_atomic */
smp_mb__after_atomic();
return (prior_state == LOCK_COUNTER_STATE_SUSPENDED);
}
/**
* vdo_resume_recovery_journal() - Resume a recovery journal which has been drained.
* @journal: The journal to resume.
* @parent: The completion to finish once the journal is resumed.
*/
void vdo_resume_recovery_journal(struct recovery_journal *journal,
struct vdo_completion *parent)
{
bool saved;
assert_on_journal_thread(journal, __func__);
saved = vdo_is_state_saved(&journal->state);
vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state));
if (is_read_only(journal)) {
vdo_continue_completion(parent, VDO_READ_ONLY);
return;
}
if (saved)
initialize_journal_state(journal);
if (resume_lock_counter(&journal->lock_counter)) {
/* We might have missed a notification. */
reap_recovery_journal(journal);
}
vdo_launch_completion(parent);
}
/**
* vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the
* VDO.
* @journal: The journal.
*
* Return: The number of logical blocks in use by the VDO.
*/
block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal)
{
return journal->logical_blocks_used;
}
/**
* vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal.
* @journal: The recovery journal to query.
*
* Return: A copy of the current statistics for the journal.
*/
struct recovery_journal_statistics
vdo_get_recovery_journal_statistics(const struct recovery_journal *journal)
{
return journal->events;
}
/**
* dump_recovery_block() - Dump the contents of the recovery block to the log.
* @block: The block to dump.
*/
static void dump_recovery_block(const struct recovery_journal_block *block)
{
vdo_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters",
(unsigned long long) block->sequence_number, block->entry_count,
(block->committing ? "committing" : "waiting"),
vdo_waitq_num_waiters(&block->entry_waiters),
vdo_waitq_num_waiters(&block->commit_waiters));
}
/**
* vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from
* the recovery journal.
* @journal: The recovery journal to dump.
*/
void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal)
{
const struct recovery_journal_block *block;
struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal);
vdo_log_info("Recovery Journal");
vdo_log_info(" block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu",
(unsigned long long) journal->block_map_head,
(unsigned long long) journal->slab_journal_head,
(unsigned long long) journal->last_write_acknowledged,
(unsigned long long) journal->tail,
(unsigned long long) journal->block_map_reap_head,
(unsigned long long) journal->slab_journal_reap_head,
(unsigned long long) stats.disk_full,
(unsigned long long) stats.slab_journal_commits_requested,
vdo_waitq_num_waiters(&journal->entry_waiters));
vdo_log_info(" entries: started=%llu written=%llu committed=%llu",
(unsigned long long) stats.entries.started,
(unsigned long long) stats.entries.written,
(unsigned long long) stats.entries.committed);
vdo_log_info(" blocks: started=%llu written=%llu committed=%llu",
(unsigned long long) stats.blocks.started,
(unsigned long long) stats.blocks.written,
(unsigned long long) stats.blocks.committed);
vdo_log_info(" active blocks:");
list_for_each_entry(block, &journal->active_tail_blocks, list_node)
dump_recovery_block(block);
}
Codebase Browser
linux