dm-writecache.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2018 Red Hat. All rights reserved.
 *
 * This file is released under the GPL.
 */

#include <linux/device-mapper.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/vmalloc.h>
#include <linux/kthread.h>
#include <linux/dm-io.h>
#include <linux/dm-kcopyd.h>
#include <linux/dax.h>
#include <linux/pfn_t.h>
#include <linux/libnvdimm.h>
#include <linux/delay.h>
#include "dm-io-tracker.h"

#define DM_MSG_PREFIX …

#define HIGH_WATERMARK …
#define LOW_WATERMARK …
#define MAX_WRITEBACK_JOBS …
#define ENDIO_LATENCY …
#define WRITEBACK_LATENCY …
#define AUTOCOMMIT_BLOCKS_SSD …
#define AUTOCOMMIT_BLOCKS_PMEM …
#define AUTOCOMMIT_MSEC …
#define MAX_AGE_DIV …
#define MAX_AGE_UNSPECIFIED …
#define PAUSE_WRITEBACK …

#define BITMAP_GRANULARITY …
#if BITMAP_GRANULARITY < PAGE_SIZE
#undef BITMAP_GRANULARITY
#define BITMAP_GRANULARITY …
#endif

#if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX)
#define DM_WRITECACHE_HAS_PMEM
#endif

#ifdef DM_WRITECACHE_HAS_PMEM
#define pmem_assign(dest, src) …
#else
#define pmem_assign …
#endif

#if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
#endif

#define MEMORY_SUPERBLOCK_MAGIC …
#define MEMORY_SUPERBLOCK_VERSION …

struct wc_memory_entry { … };

struct wc_memory_superblock { … };

struct wc_entry { … };

#ifdef DM_WRITECACHE_HAS_PMEM
#define WC_MODE_PMEM(wc) …
#define WC_MODE_FUA(wc) …
#else
#define WC_MODE_PMEM …
#define WC_MODE_FUA …
#endif
#define WC_MODE_SORT_FREELIST(wc) …

struct dm_writecache { … };

#define WB_LIST_INLINE …

struct writeback_struct { … };

struct copy_struct { … };

DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(…) …;

static void wc_lock(struct dm_writecache *wc)
{ … }

static void wc_unlock(struct dm_writecache *wc)
{ … }

#ifdef DM_WRITECACHE_HAS_PMEM
static int persistent_memory_claim(struct dm_writecache *wc)
{ … }
#else
static int persistent_memory_claim(struct dm_writecache *wc)
{
	return -EOPNOTSUPP;
}
#endif

static void persistent_memory_release(struct dm_writecache *wc)
{ … }

static struct page *persistent_memory_page(void *addr)
{ … }

static unsigned int persistent_memory_page_offset(void *addr)
{ … }

static void persistent_memory_flush_cache(void *ptr, size_t size)
{ … }

static void persistent_memory_invalidate_cache(void *ptr, size_t size)
{ … }

static struct wc_memory_superblock *sb(struct dm_writecache *wc)
{ … }

static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
					    uint64_t original_sector, uint64_t seq_count)
{ … }

#define writecache_error(wc, err, msg, arg...) …

#define writecache_has_error(wc) …

static void writecache_flush_all_metadata(struct dm_writecache *wc)
{ … }

static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
{ … }

static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);

struct io_notify { … };

static void writecache_notify_io(unsigned long error, void *context)
{ … }

static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
{ … }

static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
{ … }

static void ssd_commit_superblock(struct dm_writecache *wc)
{ … }

static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
{ … }

static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
{ … }

#define WFE_RETURN_FOLLOWING …
#define WFE_LOWEST_SEQ …

static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
					      uint64_t block, int flags)
{ … }

static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
{ … }

static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static inline void writecache_verify_watermark(struct dm_writecache *wc)
{ … }

static void writecache_max_age_timer(struct timer_list *t)
{ … }

static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
{ … }

static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void writecache_wait_on_freelist(struct dm_writecache *wc)
{ … }

static void writecache_poison_lists(struct dm_writecache *wc)
{ … }

static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
{ … }

static void writecache_flush(struct dm_writecache *wc)
{ … }

static void writecache_flush_work(struct work_struct *work)
{ … }

static void writecache_autocommit_timer(struct timer_list *t)
{ … }

static void writecache_schedule_autocommit(struct dm_writecache *wc)
{ … }

static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
{ … }

static bool writecache_wait_for_writeback(struct dm_writecache *wc)
{ … }

static void writecache_suspend(struct dm_target *ti)
{ … }

static int writecache_alloc_entries(struct dm_writecache *wc)
{ … }

static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors)
{ … }

static void writecache_resume(struct dm_target *ti)
{ … }

static int process_flush_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
{ … }

static int process_flush_on_suspend_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
{ … }

static void activate_cleaner(struct dm_writecache *wc)
{ … }

static int process_cleaner_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
{ … }

static int process_clear_stats_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
{ … }

static int writecache_message(struct dm_target *ti, unsigned int argc, char **argv,
			      char *result, unsigned int maxlen)
{ … }

static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
{ … }

static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
{ … }

static int writecache_flush_thread(void *data)
{ … }

static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
{ … }

enum wc_map_op { … };

static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
					struct wc_entry *e)
{ … }

static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
{ … }

static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
				    struct wc_entry *e, bool search_used)
{ … }

static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
{ … }

static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio)
{ … }

static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
{ … }

static int writecache_map(struct dm_target *ti, struct bio *bio)
{ … }

static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
{ … }

static int writecache_iterate_devices(struct dm_target *ti,
				      iterate_devices_callout_fn fn, void *data)
{ … }

static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
{ … }


static void writecache_writeback_endio(struct bio *bio)
{ … }

static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
{ … }

static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
{ … }

static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
{ … }

static int writecache_endio_thread(void *data)
{ … }

static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e)
{ … }

struct writeback_list { … };

static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
{ … }

static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
{ … }

static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
{ … }

static void writecache_writeback(struct work_struct *work)
{ … }

static int calculate_memory_size(uint64_t device_size, unsigned int block_size,
				 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
{ … }

static int init_memory(struct dm_writecache *wc)
{ … }

static void writecache_dtr(struct dm_target *ti)
{ … }

static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{ … }

static void writecache_status(struct dm_target *ti, status_type_t type,
			      unsigned int status_flags, char *result, unsigned int maxlen)
{ … }

static struct target_type writecache_target = …;
module_dm(writecache);

MODULE_DESCRIPTION(…) …;
MODULE_AUTHOR(…) …;
MODULE_LICENSE(…) …;
linux/drivers/md/dm-writecache.c
