// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017 Western Digital Corporation or its affiliates. * * This file is released under the GPL. */ #include "dm-zoned.h" #include <linux/module.h> #include <linux/crc32.h> #include <linux/sched/mm.h> #define DM_MSG_PREFIX … /* * Metadata version. */ #define DMZ_META_VER … /* * On-disk super block magic. */ #define DMZ_MAGIC … /* * On disk super block. * This uses only 512 B but uses on disk a full 4KB block. This block is * followed on disk by the mapping table of chunks to zones and the bitmap * blocks indicating zone block validity. * The overall resulting metadata format is: * (1) Super block (1 block) * (2) Chunk mapping table (nr_map_blocks) * (3) Bitmap blocks (nr_bitmap_blocks) * All metadata blocks are stored in conventional zones, starting from * the first conventional zone found on disk. */ struct dmz_super { … }; /* * Chunk mapping entry: entries are indexed by chunk number * and give the zone ID (dzone_id) mapping the chunk on disk. * This zone may be sequential or random. If it is a sequential * zone, a second zone (bzone_id) used as a write buffer may * also be specified. This second zone will always be a randomly * writeable zone. */ struct dmz_map { … }; /* * Chunk mapping table metadata: 512 8-bytes entries per 4KB block. */ #define DMZ_MAP_ENTRIES … #define DMZ_MAP_ENTRIES_SHIFT … #define DMZ_MAP_ENTRIES_MASK … #define DMZ_MAP_UNMAPPED … /* * Meta data block descriptor (for cached metadata blocks). */ struct dmz_mblock { … }; /* * Metadata block state flags. */ enum { … }; /* * Super block information (one per metadata set). */ struct dmz_sb { … }; /* * In-memory metadata. */ struct dmz_metadata { … }; #define dmz_zmd_info(zmd, format, args...) … #define dmz_zmd_err(zmd, format, args...) … #define dmz_zmd_warn(zmd, format, args...) … #define dmz_zmd_debug(zmd, format, args...) … /* * Various accessors */ static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone) { … } sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone) { … } sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone) { … } unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd) { … } unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd) { … } unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd) { … } unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd) { … } unsigned int dmz_nr_zones(struct dmz_metadata *zmd) { … } unsigned int dmz_nr_chunks(struct dmz_metadata *zmd) { … } unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx) { … } unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx) { … } unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd) { … } unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd) { … } unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx) { … } unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx) { … } static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id) { … } static struct dm_zone *dmz_insert(struct dmz_metadata *zmd, unsigned int zone_id, struct dmz_dev *dev) { … } const char *dmz_metadata_label(struct dmz_metadata *zmd) { … } bool dmz_check_dev(struct dmz_metadata *zmd) { … } bool dmz_dev_is_dying(struct dmz_metadata *zmd) { … } /* * Lock/unlock mapping table. * The map lock also protects all the zone lists. */ void dmz_lock_map(struct dmz_metadata *zmd) { … } void dmz_unlock_map(struct dmz_metadata *zmd) { … } /* * Lock/unlock metadata access. This is a "read" lock on a semaphore * that prevents metadata flush from running while metadata are being * modified. The actual metadata write mutual exclusion is achieved with * the map lock and zone state management (active and reclaim state are * mutually exclusive). */ void dmz_lock_metadata(struct dmz_metadata *zmd) { … } void dmz_unlock_metadata(struct dmz_metadata *zmd) { … } /* * Lock/unlock flush: prevent concurrent executions * of dmz_flush_metadata as well as metadata modification in reclaim * while flush is being executed. */ void dmz_lock_flush(struct dmz_metadata *zmd) { … } void dmz_unlock_flush(struct dmz_metadata *zmd) { … } /* * Allocate a metadata block. */ static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd, sector_t mblk_no) { … } /* * Free a metadata block. */ static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk) { … } /* * Insert a metadata block in the rbtree. */ static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk) { … } /* * Lookup a metadata block in the rbtree. If the block is found, increment * its reference count. */ static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd, sector_t mblk_no) { … } /* * Metadata block BIO end callback. */ static void dmz_mblock_bio_end_io(struct bio *bio) { … } /* * Read an uncached metadata block from disk and add it to the cache. */ static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd, sector_t mblk_no) { … } /* * Free metadata blocks. */ static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd, unsigned long limit) { … } /* * For mblock shrinker: get the number of unused metadata blocks in the cache. */ static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) { … } /* * For mblock shrinker: scan unused metadata blocks and shrink the cache. */ static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { … } /* * Release a metadata block. */ static void dmz_release_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk) { … } /* * Get a metadata block from the rbtree. If the block * is not present, read it from disk. */ static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd, sector_t mblk_no) { … } /* * Mark a metadata block dirty. */ static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk) { … } /* * Issue a metadata block write BIO. */ static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk, unsigned int set) { … } /* * Read/write a metadata block. */ static int dmz_rdwr_block(struct dmz_dev *dev, enum req_op op, sector_t block, struct page *page) { … } /* * Write super block of the specified metadata set. */ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) { … } /* * Write dirty metadata blocks to the specified set. */ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd, struct list_head *write_list, unsigned int set) { … } /* * Log dirty metadata blocks. */ static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd, struct list_head *write_list) { … } /* * Flush dirty metadata blocks. */ int dmz_flush_metadata(struct dmz_metadata *zmd) { … } /* * Check super block. */ static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb, bool tertiary) { … } /* * Read the first or second super block from disk. */ static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set) { … } /* * Determine the position of the secondary super blocks on disk. * This is used only if a corruption of the primary super block * is detected. */ static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd) { … } /* * Read a super block from disk. */ static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set) { … } /* * Recover a metadata set. */ static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set) { … } /* * Get super block from disk. */ static int dmz_load_sb(struct dmz_metadata *zmd) { … } /* * Initialize a zone descriptor. */ static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data) { … } static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev) { … } /* * Free zones descriptors. */ static void dmz_drop_zones(struct dmz_metadata *zmd) { … } /* * Allocate and initialize zone descriptors using the zone * information from disk. */ static int dmz_init_zones(struct dmz_metadata *zmd) { … } static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx, void *data) { … } /* * Update a zone information. */ static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Check a zone write pointer position when the zone is marked * with the sequential write error flag. */ static int dmz_handle_seq_write_err(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Reset a zone write pointer. */ static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone); /* * Initialize chunk mapping. */ static int dmz_load_mapping(struct dmz_metadata *zmd) { … } /* * Set a data chunk mapping. */ static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, unsigned int dzone_id, unsigned int bzone_id) { … } /* * The list of mapped zones is maintained in LRU order. * This rotates a zone at the end of its map list. */ static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * The list of mapped random zones is maintained * in LRU order. This rotates a zone at the end of the list. */ static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Wait for any zone to be freed. */ static void dmz_wait_for_free_zones(struct dmz_metadata *zmd) { … } /* * Lock a zone for reclaim (set the zone RECLAIM bit). * Returns false if the zone cannot be locked or if it is already locked * and 1 otherwise. */ int dmz_lock_zone_reclaim(struct dm_zone *zone) { … } /* * Clear a zone reclaim flag. */ void dmz_unlock_zone_reclaim(struct dm_zone *zone) { … } /* * Wait for a zone reclaim to complete. */ static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Select a cache or random write zone for reclaim. */ static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd, unsigned int idx, bool idle) { … } /* * Select a buffered sequential zone for reclaim. */ static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd, unsigned int idx) { … } /* * Select a zone for reclaim. */ struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd, unsigned int dev_idx, bool idle) { … } /* * Get the zone mapping a chunk, if the chunk is mapped already. * If no mapping exist and the operation is WRITE, a zone is * allocated and used to map the chunk. * The zone returned will be set to the active state. */ struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, enum req_op op) { … } /* * Write and discard change the block validity of data zones and their buffer * zones. Check here that valid blocks are still present. If all blocks are * invalid, the zones can be unmapped on the fly without waiting for reclaim * to do it. */ void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone) { … } /* * Allocate and map a random zone to buffer a chunk * already mapped to a sequential zone. */ struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd, struct dm_zone *dzone) { … } /* * Get an unmapped (free) zone. * This must be called with the mapping lock held. */ struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx, unsigned long flags) { … } /* * Free a zone. * This must be called with the mapping lock held. */ void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Map a chunk to a zone. * This must be called with the mapping lock held. */ void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone, unsigned int chunk) { … } /* * Unmap a zone. * This must be called with the mapping lock held. */ void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Set @nr_bits bits in @bitmap starting from @bit. * Return the number of bits changed from 0 to 1. */ static unsigned int dmz_set_bits(unsigned long *bitmap, unsigned int bit, unsigned int nr_bits) { … } /* * Get the bitmap block storing the bit for chunk_block in zone. */ static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block) { … } /* * Copy the valid blocks bitmap of from_zone to the bitmap of to_zone. */ int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone, struct dm_zone *to_zone) { … } /* * Merge the valid blocks bitmap of from_zone into the bitmap of to_zone, * starting from chunk_block. */ int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone, struct dm_zone *to_zone, sector_t chunk_block) { … } /* * Validate all the blocks in the range [block..block+nr_blocks-1]. */ int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block, unsigned int nr_blocks) { … } /* * Clear nr_bits bits in bitmap starting from bit. * Return the number of bits cleared. */ static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits) { … } /* * Invalidate all the blocks in the range [block..block+nr_blocks-1]. */ int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block, unsigned int nr_blocks) { … } /* * Get a block bit value. */ static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block) { … } /* * Return the number of blocks from chunk_block to the first block with a bit * value specified by set. Search at most nr_blocks blocks from chunk_block. */ static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block, unsigned int nr_blocks, int set) { … } /* * Test if chunk_block is valid. If it is, the number of consecutive * valid blocks from chunk_block will be returned. */ int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t chunk_block) { … } /* * Find the first valid block from @chunk_block in @zone. * If such a block is found, its number is returned using * @chunk_block and the total number of valid blocks from @chunk_block * is returned. */ int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone, sector_t *chunk_block) { … } /* * Count the number of bits set starting from bit up to bit + nr_bits - 1. */ static int dmz_count_bits(void *bitmap, int bit, int nr_bits) { … } /* * Get a zone weight. */ static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone) { … } /* * Cleanup the zoned metadata resources. */ static void dmz_cleanup_metadata(struct dmz_metadata *zmd) { … } static void dmz_print_dev(struct dmz_metadata *zmd, int num) { … } /* * Initialize the zoned metadata. */ int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev, struct dmz_metadata **metadata, const char *devname) { … } /* * Cleanup the zoned metadata resources. */ void dmz_dtr_metadata(struct dmz_metadata *zmd) { … } /* * Check zone information on resume. */ int dmz_resume_metadata(struct dmz_metadata *zmd) { … }
Codebase Browser
linux