// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved. * Copyright (C) 2016-2017 Milan Broz * Copyright (C) 2016-2017 Mikulas Patocka * * This file is released under the GPL. */ #include "dm-bio-record.h" #include <linux/compiler.h> #include <linux/module.h> #include <linux/device-mapper.h> #include <linux/dm-io.h> #include <linux/vmalloc.h> #include <linux/sort.h> #include <linux/rbtree.h> #include <linux/delay.h> #include <linux/random.h> #include <linux/reboot.h> #include <crypto/hash.h> #include <crypto/skcipher.h> #include <linux/async_tx.h> #include <linux/dm-bufio.h> #include "dm-audit.h" #define DM_MSG_PREFIX … #define DEFAULT_INTERLEAVE_SECTORS … #define DEFAULT_JOURNAL_SIZE_FACTOR … #define DEFAULT_SECTORS_PER_BITMAP_BIT … #define DEFAULT_BUFFER_SECTORS … #define DEFAULT_JOURNAL_WATERMARK … #define DEFAULT_SYNC_MSEC … #define DEFAULT_MAX_JOURNAL_SECTORS … #define MIN_LOG2_INTERLEAVE_SECTORS … #define MAX_LOG2_INTERLEAVE_SECTORS … #define METADATA_WORKQUEUE_MAX_ACTIVE … #define RECALC_SECTORS … #define RECALC_WRITE_SUPER … #define BITMAP_BLOCK_SIZE … #define BITMAP_FLUSH_INTERVAL … #define DISCARD_FILLER … #define SALT_SIZE … #define RECHECK_POOL_SIZE … /* * Warning - DEBUG_PRINT prints security-sensitive data to the log, * so it should not be enabled in the official kernel */ //#define DEBUG_PRINT //#define INTERNAL_VERIFY /* * On disk structures */ #define SB_MAGIC … #define SB_VERSION_1 … #define SB_VERSION_2 … #define SB_VERSION_3 … #define SB_VERSION_4 … #define SB_VERSION_5 … #define SB_VERSION_6 … #define SB_SECTORS … #define MAX_SECTORS_PER_BLOCK … struct superblock { … }; #define SB_FLAG_HAVE_JOURNAL_MAC … #define SB_FLAG_RECALCULATING … #define SB_FLAG_DIRTY_BITMAP … #define SB_FLAG_FIXED_PADDING … #define SB_FLAG_FIXED_HMAC … #define SB_FLAG_INLINE … #define JOURNAL_ENTRY_ROUNDUP … commit_id_t; #define JOURNAL_MAC_PER_SECTOR … struct journal_entry { … }; #define journal_entry_tag(ic, je) … #if BITS_PER_LONG == 64 #define journal_entry_set_sector(je, x) … #else #define journal_entry_set_sector … #endif #define journal_entry_get_sector(je) … #define journal_entry_is_unused(je) … #define journal_entry_set_unused(je) … #define journal_entry_is_inprogress(je) … #define journal_entry_set_inprogress(je) … #define JOURNAL_BLOCK_SECTORS … #define JOURNAL_SECTOR_DATA … #define JOURNAL_MAC_SIZE … struct journal_sector { … }; #define MAX_TAG_SIZE … #define METADATA_PADDING_SECTORS … #define N_COMMIT_IDS … static unsigned char prev_commit_seq(unsigned char seq) { … } static unsigned char next_commit_seq(unsigned char seq) { … } /* * In-memory structures */ struct journal_node { … }; struct alg_spec { … }; struct dm_integrity_c { … }; struct dm_integrity_range { … }; struct dm_integrity_io { … }; struct journal_completion { … }; struct journal_io { … }; struct bitmap_block_status { … }; static struct kmem_cache *journal_io_cache; #define JOURNAL_IO_MEMPOOL … #ifdef DEBUG_PRINT #define DEBUG_print … #define DEBUG_bytes … #else #define DEBUG_print(x, ...) … #define DEBUG_bytes(bytes, len, msg, ...) … #endif static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); static int dm_integrity_map_inline(struct dm_integrity_io *dio); static void integrity_bio_wait(struct work_struct *w); static void dm_integrity_dtr(struct dm_target *ti); static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) { … } static int dm_integrity_failed(struct dm_integrity_c *ic) { … } static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic) { … } static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i, unsigned int j, unsigned char seq) { … } static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, sector_t *area, sector_t *offset) { … } #define sector_to_block(ic, n) … static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, sector_t offset, unsigned int *metadata_offset) { … } static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset) { … } static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) { … } static void sb_set_version(struct dm_integrity_c *ic) { … } static int sb_mac(struct dm_integrity_c *ic, bool wr) { … } static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) { … } #define BITMAP_OP_TEST_ALL_SET … #define BITMAP_OP_TEST_ALL_CLEAR … #define BITMAP_OP_SET … #define BITMAP_OP_CLEAR … static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap, sector_t sector, sector_t n_sectors, int mode) { … } static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src) { … } static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector) { … } static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, bool e, const char *function) { … } static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, unsigned int *pl_index, unsigned int *pl_offset) { … } static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, unsigned int section, unsigned int offset, unsigned int *n_sectors) { … } static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset) { … } static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, unsigned int n) { … } static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, unsigned int n) { … } static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SIZE]) { … } static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr) { … } static void complete_journal_op(void *context) { … } static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, unsigned int n_sections, struct journal_completion *comp) { … } static void complete_journal_encrypt(void *data, int err) { … } static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp) { … } static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, unsigned int n_sections, struct journal_completion *comp) { … } static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, unsigned int n_sections, struct journal_completion *comp) { … } static void complete_journal_io(unsigned long error, void *context) { … } static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, unsigned int sector, unsigned int n_sectors, struct journal_completion *comp) { … } static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf, unsigned int section, unsigned int n_sections, struct journal_completion *comp) { … } static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit_sections) { … } static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, unsigned int n_sectors, sector_t target, io_notify_fn fn, void *data) { … } static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2) { … } static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting) { … } static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) { … } static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) { … } static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) { … } static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) { … } static void init_journal_node(struct journal_node *node) { … } static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector) { … } static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node) { … } #define NOT_FOUND … static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) { … } static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector) { … } static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node) { … } #define TAG_READ … #define TAG_WRITE … #define TAG_CMP … static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block, unsigned int *metadata_offset, unsigned int total_size, int op) { … } struct flush_request { … }; static void flush_notify(unsigned long error, void *fr_) { … } static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data) { … } static void sleep_on_endio_wait(struct dm_integrity_c *ic) { … } static void autocommit_fn(struct timer_list *t) { … } static void schedule_autocommit(struct dm_integrity_c *ic) { … } static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) { … } static void do_endio(struct dm_integrity_c *ic, struct bio *bio) { … } static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) { … } static void dec_in_flight(struct dm_integrity_io *dio) { … } static void integrity_end_io(struct bio *bio) { … } static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, const char *data, char *result) { … } static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) { … } static void integrity_metadata(struct work_struct *w) { … } static inline bool dm_integrity_check_limits(struct dm_integrity_c *ic, sector_t logical_sector, struct bio *bio) { … } static int dm_integrity_map(struct dm_target *ti, struct bio *bio) { … } static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, unsigned int journal_section, unsigned int journal_entry) { … } static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) { … } static int dm_integrity_map_inline(struct dm_integrity_io *dio) { … } static inline void dm_integrity_free_payload(struct dm_integrity_io *dio) { … } static void dm_integrity_inline_recheck(struct work_struct *w) { … } static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) { … } static void integrity_bio_wait(struct work_struct *w) { … } static void pad_uncommitted(struct dm_integrity_c *ic) { … } static void integrity_commit(struct work_struct *w) { … } static void complete_copy_from_journal(unsigned long error, void *context) { … } static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, struct journal_entry *je) { … } static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start, unsigned int write_sections, bool from_replay) { … } static void integrity_writer(struct work_struct *w) { … } static void recalc_write_super(struct dm_integrity_c *ic) { … } static void integrity_recalc(struct work_struct *w) { … } static void bitmap_block_work(struct work_struct *w) { … } static void bitmap_flush_work(struct work_struct *work) { … } static void init_journal(struct dm_integrity_c *ic, unsigned int start_section, unsigned int n_sections, unsigned char commit_seq) { … } static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t id) { … } static void replay_journal(struct dm_integrity_c *ic) { … } static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic) { … } static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x) { … } static void dm_integrity_postsuspend(struct dm_target *ti) { … } static void dm_integrity_resume(struct dm_target *ti) { … } static void dm_integrity_status(struct dm_target *ti, status_type_t type, unsigned int status_flags, char *result, unsigned int maxlen) { … } static int dm_integrity_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { … } static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits) { … } static void calculate_journal_section_size(struct dm_integrity_c *ic) { … } static int calculate_device_limits(struct dm_integrity_c *ic) { … } static void get_provided_data_sectors(struct dm_integrity_c *ic) { … } static int initialize_superblock(struct dm_integrity_c *ic, unsigned int journal_sectors, unsigned int interleave_sectors) { … } static void dm_integrity_free_page_list(struct page_list *pl) { … } static struct page_list *dm_integrity_alloc_page_list(unsigned int n_pages) { … } static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl) { … } static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl) { … } static void free_alg(struct alg_spec *a) { … } static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval) { … } static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, char *error_alg, char *error_key) { … } static int create_journal(struct dm_integrity_c *ic, char **error) { … } /* * Construct a integrity mapping * * Arguments: * device * offset from the start of the device * tag size * D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode * number of optional arguments * optional arguments: * journal_sectors * interleave_sectors * buffer_sectors * journal_watermark * commit_time * meta_device * block_size * sectors_per_bit * bitmap_flush_interval * internal_hash * journal_crypt * journal_mac * recalculate */ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv) { … } static void dm_integrity_dtr(struct dm_target *ti) { … } static struct target_type integrity_target = …; static int __init dm_integrity_init(void) { … } static void __exit dm_integrity_exit(void) { … } module_init(…) …; module_exit(dm_integrity_exit); MODULE_AUTHOR(…) …; MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux