linux/drivers/md/dm-vdo/flush.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2023 Red Hat
 */

#include "flush.h"

#include <linux/mempool.h>
#include <linux/spinlock.h>

#include "logger.h"
#include "memory-alloc.h"
#include "permassert.h"

#include "admin-state.h"
#include "completion.h"
#include "io-submitter.h"
#include "logical-zone.h"
#include "slab-depot.h"
#include "types.h"
#include "vdo.h"

struct flusher { … };

/**
 * assert_on_flusher_thread() - Check that we are on the flusher thread.
 * @flusher: The flusher.
 * @caller: The function which is asserting.
 */
static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller)
{ … }

/**
 * as_flusher() - Convert a generic vdo_completion to a flusher.
 * @completion: The completion to convert.
 *
 * Return: The completion as a flusher.
 */
static struct flusher *as_flusher(struct vdo_completion *completion)
{ … }

/**
 * completion_as_vdo_flush() - Convert a generic vdo_completion to a vdo_flush.
 * @completion: The completion to convert.
 *
 * Return: The completion as a vdo_flush.
 */
static inline struct vdo_flush *completion_as_vdo_flush(struct vdo_completion *completion)
{ … }

/**
 * vdo_waiter_as_flush() - Convert a vdo_flush's generic wait queue entry back to the vdo_flush.
 * @waiter: The wait queue entry to convert.
 *
 * Return: The wait queue entry as a vdo_flush.
 */
static struct vdo_flush *vdo_waiter_as_flush(struct vdo_waiter *waiter)
{ … }

static void *allocate_flush(gfp_t gfp_mask, void *pool_data)
{ … }

static void free_flush(void *element, void *pool_data __always_unused)
{ … }

/**
 * vdo_make_flusher() - Make a flusher for a vdo.
 * @vdo: The vdo which owns the flusher.
 *
 * Return: VDO_SUCCESS or an error.
 */
int vdo_make_flusher(struct vdo *vdo)
{ … }

/**
 * vdo_free_flusher() - Free a flusher.
 * @flusher: The flusher to free.
 */
void vdo_free_flusher(struct flusher *flusher)
{ … }

/**
 * vdo_get_flusher_thread_id() - Get the ID of the thread on which flusher functions should be
 *                               called.
 * @flusher: The flusher to query.
 *
 * Return: The ID of the thread which handles the flusher.
 */
thread_id_t vdo_get_flusher_thread_id(struct flusher *flusher)
{ … }

static void notify_flush(struct flusher *flusher);
static void vdo_complete_flush(struct vdo_flush *flush);

/**
 * finish_notification() - Finish the notification process.
 * @completion: The flusher completion.
 *
 * Finishes the notification process by checking if any flushes have completed and then starting
 * the notification of the next flush request if one came in while the current notification was in
 * progress. This callback is registered in flush_packer_callback().
 */
static void finish_notification(struct vdo_completion *completion)
{ … }

/**
 * flush_packer_callback() - Flush the packer.
 * @completion: The flusher completion.
 *
 * Flushes the packer now that all of the logical and physical zones have been notified of the new
 * flush request. This callback is registered in increment_generation().
 */
static void flush_packer_callback(struct vdo_completion *completion)
{ … }

/**
 * increment_generation() - Increment the flush generation in a logical zone.
 * @completion: The flusher as a completion.
 *
 * If there are more logical zones, go on to the next one, otherwise, prepare the physical zones.
 * This callback is registered both in notify_flush() and in itself.
 */
static void increment_generation(struct vdo_completion *completion)
{ … }

/**
 * notify_flush() - Launch a flush notification.
 * @flusher: The flusher doing the notification.
 */
static void notify_flush(struct flusher *flusher)
{ … }

/**
 * flush_vdo() - Start processing a flush request.
 * @completion: A flush request (as a vdo_completion)
 *
 * This callback is registered in launch_flush().
 */
static void flush_vdo(struct vdo_completion *completion)
{ … }

/**
 * check_for_drain_complete() - Check whether the flusher has drained.
 * @flusher: The flusher.
 */
static void check_for_drain_complete(struct flusher *flusher)
{ … }

/**
 * vdo_complete_flushes() - Attempt to complete any flushes which might have finished.
 * @flusher: The flusher.
 */
void vdo_complete_flushes(struct flusher *flusher)
{ … }

/**
 * vdo_dump_flusher() - Dump the flusher, in a thread-unsafe fashion.
 * @flusher: The flusher.
 */
void vdo_dump_flusher(const struct flusher *flusher)
{ … }

/**
 * initialize_flush() - Initialize a vdo_flush structure.
 * @flush: The flush to initialize.
 * @vdo: The vdo being flushed.
 *
 * Initializes a vdo_flush structure, transferring all the bios in the flusher's waiting_flush_bios
 * list to it. The caller MUST already hold the lock.
 */
static void initialize_flush(struct vdo_flush *flush, struct vdo *vdo)
{ … }

static void launch_flush(struct vdo_flush *flush)
{ … }

/**
 * vdo_launch_flush() - Function called to start processing a flush request.
 * @vdo: The vdo.
 * @bio: The bio containing an empty flush request.
 *
 * This is called when we receive an empty flush bio from the block layer, and before acknowledging
 * a non-empty bio with the FUA flag set.
 */
void vdo_launch_flush(struct vdo *vdo, struct bio *bio)
{ … }

/**
 * release_flush() - Release a vdo_flush structure that has completed its work.
 * @flush: The completed flush structure to re-use or free.
 *
 * If there are any pending flush requests whose vdo_flush allocation failed, they will be launched
 * by immediately re-using the released vdo_flush. If there is no spare vdo_flush, the released
 * structure will become the spare. Otherwise, the vdo_flush will be freed.
 */
static void release_flush(struct vdo_flush *flush)
{ … }

/**
 * vdo_complete_flush_callback() - Function called to complete and free a flush request, registered
 *                                 in vdo_complete_flush().
 * @completion: The flush request.
 */
static void vdo_complete_flush_callback(struct vdo_completion *completion)
{ … }

/**
 * select_bio_queue() - Select the bio queue on which to finish a flush request.
 * @flusher: The flusher finishing the request.
 */
static thread_id_t select_bio_queue(struct flusher *flusher)
{ … }

/**
 * vdo_complete_flush() - Complete and free a vdo flush request.
 * @flush: The flush request.
 */
static void vdo_complete_flush(struct vdo_flush *flush)
{ … }

/**
 * initiate_drain() - Initiate a drain.
 *
 * Implements vdo_admin_initiator_fn.
 */
static void initiate_drain(struct admin_state *state)
{ … }

/**
 * vdo_drain_flusher() - Drain the flusher.
 * @flusher: The flusher to drain.
 * @completion: The completion to finish when the flusher has drained.
 *
 * Drains the flusher by preventing any more VIOs from entering the flusher and then flushing. The
 * flusher will be left in the suspended state.
 */
void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion)
{ … }

/**
 * vdo_resume_flusher() - Resume a flusher which has been suspended.
 * @flusher: The flusher to resume.
 * @parent: The completion to finish when the flusher has resumed.
 */
void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent)
{ … }