// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved. */ #include <linux/mm.h> #include <linux/err.h> #include <linux/slab.h> #include <linux/rwsem.h> #include <linux/bitops.h> #include <linux/bitmap.h> #include <linux/device-mapper.h> #include "persistent-data/dm-bitset.h" #include "persistent-data/dm-space-map.h" #include "persistent-data/dm-block-manager.h" #include "persistent-data/dm-transaction-manager.h" #include "dm-clone-metadata.h" #define DM_MSG_PREFIX … #define SUPERBLOCK_LOCATION … #define SUPERBLOCK_MAGIC … #define SUPERBLOCK_CSUM_XOR … #define DM_CLONE_MAX_CONCURRENT_LOCKS … #define UUID_LEN … /* Min and max dm-clone metadata versions supported */ #define DM_CLONE_MIN_METADATA_VERSION … #define DM_CLONE_MAX_METADATA_VERSION … /* * On-disk metadata layout */ struct superblock_disk { … } __packed; /* * Region and Dirty bitmaps. * * dm-clone logically splits the source and destination devices in regions of * fixed size. The destination device's regions are gradually hydrated, i.e., * we copy (clone) the source's regions to the destination device. Eventually, * all regions will get hydrated and all I/O will be served from the * destination device. * * We maintain an on-disk bitmap which tracks the state of each of the * destination device's regions, i.e., whether they are hydrated or not. * * To save constantly doing look ups on disk we keep an in core copy of the * on-disk bitmap, the region_map. * * In order to track which regions are hydrated during a metadata transaction, * we use a second set of bitmaps, the dmap (dirty bitmap), which includes two * bitmaps, namely dirty_regions and dirty_words. The dirty_regions bitmap * tracks the regions that got hydrated during the current metadata * transaction. The dirty_words bitmap tracks the dirty words, i.e. longs, of * the dirty_regions bitmap. * * This allows us to precisely track the regions that were hydrated during the * current metadata transaction and update the metadata accordingly, when we * commit the current transaction. This is important because dm-clone should * only commit the metadata of regions that were properly flushed to the * destination device beforehand. Otherwise, in case of a crash, we could end * up with a corrupted dm-clone device. * * When a region finishes hydrating dm-clone calls * dm_clone_set_region_hydrated(), or for discard requests * dm_clone_cond_set_range(), which sets the corresponding bits in region_map * and dmap. * * During a metadata commit we scan dmap->dirty_words and dmap->dirty_regions * and update the on-disk metadata accordingly. Thus, we don't have to flush to * disk the whole region_map. We can just flush the dirty region_map bits. * * We use the helper dmap->dirty_words bitmap, which is smaller than the * original region_map, to reduce the amount of memory accesses during a * metadata commit. Moreover, as dm-bitset also accesses the on-disk bitmap in * 64-bit word granularity, the dirty_words bitmap helps us avoid useless disk * accesses. * * We could update directly the on-disk bitmap, when dm-clone calls either * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), buts this * inserts significant metadata I/O overhead in dm-clone's I/O path. Also, as * these two functions don't block, we can call them in interrupt context, * e.g., in a hooked overwrite bio's completion routine, and further reduce the * I/O completion latency. * * We maintain two dirty bitmap sets. During a metadata commit we atomically * swap the currently used dmap with the unused one. This allows the metadata * update functions to run concurrently with an ongoing commit. */ struct dirty_map { … }; struct dm_clone_metadata { … }; /*---------------------------------------------------------------------------*/ /* * Superblock validation. */ static void sb_prepare_for_write(const struct dm_block_validator *v, struct dm_block *b, size_t sb_block_size) { … } static int sb_check(const struct dm_block_validator *v, struct dm_block *b, size_t sb_block_size) { … } static const struct dm_block_validator sb_validator = …; /* * Check if the superblock is formatted or not. We consider the superblock to * be formatted in case we find non-zero bytes in it. */ static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *formatted) { … } /*---------------------------------------------------------------------------*/ /* * Low-level metadata handling. */ static inline int superblock_read_lock(struct dm_clone_metadata *cmd, struct dm_block **sblock) { … } static inline int superblock_write_lock_zero(struct dm_clone_metadata *cmd, struct dm_block **sblock) { … } static int __copy_sm_root(struct dm_clone_metadata *cmd) { … } /* Save dm-clone metadata in superblock */ static void __prepare_superblock(struct dm_clone_metadata *cmd, struct superblock_disk *sb) { … } static int __open_metadata(struct dm_clone_metadata *cmd) { … } static int __format_metadata(struct dm_clone_metadata *cmd) { … } static int __open_or_format_metadata(struct dm_clone_metadata *cmd, bool may_format_device) { … } static int __create_persistent_data_structures(struct dm_clone_metadata *cmd, bool may_format_device) { … } static void __destroy_persistent_data_structures(struct dm_clone_metadata *cmd) { … } /*---------------------------------------------------------------------------*/ static int __dirty_map_init(struct dirty_map *dmap, unsigned long nr_words, unsigned long nr_regions) { … } static void __dirty_map_exit(struct dirty_map *dmap) { … } static int dirty_map_init(struct dm_clone_metadata *cmd) { … } static void dirty_map_exit(struct dm_clone_metadata *cmd) { … } static int __load_bitset_in_core(struct dm_clone_metadata *cmd) { … } struct dm_clone_metadata *dm_clone_metadata_open(struct block_device *bdev, sector_t target_size, sector_t region_size) { … } void dm_clone_metadata_close(struct dm_clone_metadata *cmd) { … } bool dm_clone_is_hydration_done(struct dm_clone_metadata *cmd) { … } bool dm_clone_is_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr) { … } bool dm_clone_is_range_hydrated(struct dm_clone_metadata *cmd, unsigned long start, unsigned long nr_regions) { … } unsigned int dm_clone_nr_of_hydrated_regions(struct dm_clone_metadata *cmd) { … } unsigned long dm_clone_find_next_unhydrated_region(struct dm_clone_metadata *cmd, unsigned long start) { … } static int __update_metadata_word(struct dm_clone_metadata *cmd, unsigned long *dirty_regions, unsigned long word) { … } static int __metadata_commit(struct dm_clone_metadata *cmd) { … } static int __flush_dmap(struct dm_clone_metadata *cmd, struct dirty_map *dmap) { … } int dm_clone_metadata_pre_commit(struct dm_clone_metadata *cmd) { … } int dm_clone_metadata_commit(struct dm_clone_metadata *cmd) { … } int dm_clone_set_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr) { … } int dm_clone_cond_set_range(struct dm_clone_metadata *cmd, unsigned long start, unsigned long nr_regions) { … } /* * WARNING: This must not be called concurrently with either * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), as it changes * cmd->region_map without taking the cmd->bitmap_lock spinlock. The only * exception is after setting the metadata to read-only mode, using * dm_clone_metadata_set_read_only(). * * We don't take the spinlock because __load_bitset_in_core() does I/O, so it * may block. */ int dm_clone_reload_in_core_bitset(struct dm_clone_metadata *cmd) { … } bool dm_clone_changed_this_transaction(struct dm_clone_metadata *cmd) { … } int dm_clone_metadata_abort(struct dm_clone_metadata *cmd) { … } void dm_clone_metadata_set_read_only(struct dm_clone_metadata *cmd) { … } void dm_clone_metadata_set_read_write(struct dm_clone_metadata *cmd) { … } int dm_clone_get_free_metadata_block_count(struct dm_clone_metadata *cmd, dm_block_t *result) { … } int dm_clone_get_metadata_dev_size(struct dm_clone_metadata *cmd, dm_block_t *result) { … }
Codebase Browser
linux