// SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/data.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/buffer_head.h> #include <linux/sched/mm.h> #include <linux/mpage.h> #include <linux/writeback.h> #include <linux/pagevec.h> #include <linux/blkdev.h> #include <linux/bio.h> #include <linux/blk-crypto.h> #include <linux/swap.h> #include <linux/prefetch.h> #include <linux/uio.h> #include <linux/sched/signal.h> #include <linux/fiemap.h> #include <linux/iomap.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "iostat.h" #include <trace/events/f2fs.h> #define NUM_PREALLOC_POST_READ_CTXS … static struct kmem_cache *bio_post_read_ctx_cache; static struct kmem_cache *bio_entry_slab; static mempool_t *bio_post_read_ctx_pool; static struct bio_set f2fs_bioset; #define F2FS_BIO_POOL_SIZE … int __init f2fs_init_bioset(void) { … } void f2fs_destroy_bioset(void) { … } bool f2fs_is_cp_guaranteed(struct page *page) { … } static enum count_type __read_io_type(struct page *page) { … } /* postprocessing steps for read bios */ enum bio_post_read_step { … }; struct bio_post_read_ctx { … }; /* * Update and unlock a bio's pages, and free the bio. * * This marks pages up-to-date only if there was no error in the bio (I/O error, * decryption error, or verity error), as indicated by bio->bi_status. * * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) * aren't marked up-to-date here, as decompression is done on a per-compression- * cluster basis rather than a per-bio basis. Instead, we only must do two * things for each compressed page here: call f2fs_end_read_compressed_page() * with failed=true if an error occurred before it would have normally gotten * called (i.e., I/O error or decryption error, but *not* verity error), and * release the bio's reference to the decompress_io_ctx of the page's cluster. */ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) { … } static void f2fs_verify_bio(struct work_struct *work) { … } /* * If the bio's data needs to be verified with fs-verity, then enqueue the * verity work for the bio. Otherwise finish the bio now. * * Note that to avoid deadlocks, the verity work can't be done on the * decryption/decompression workqueue. This is because verifying the data pages * can involve reading verity metadata pages from the file, and these verity * metadata pages may be encrypted and/or compressed. */ static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) { … } /* * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last * remaining page was read by @ctx->bio. * * Note that a bio may span clusters (even a mix of compressed and uncompressed * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates * that the bio includes at least one compressed page. The actual decompression * is done on a per-cluster basis, not a per-bio basis. */ static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, bool in_task) { … } static void f2fs_post_read_work(struct work_struct *work) { … } static void f2fs_read_end_io(struct bio *bio) { … } static void f2fs_write_end_io(struct bio *bio) { … } #ifdef CONFIG_BLK_DEV_ZONED static void f2fs_zone_write_end_io(struct bio *bio) { … } #endif struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, block_t blk_addr, sector_t *sector) { … } int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) { … } static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) { … } static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) { … } static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask) { … } static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, pgoff_t next_idx, const struct f2fs_io_info *fio) { … } void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { … } static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, enum page_type type) { … } static void __submit_merged_bio(struct f2fs_bio_info *io) { … } static bool __has_merged_page(struct bio *bio, struct inode *inode, struct page *page, nid_t ino) { … } int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) { … } static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type, enum temp_type temp) { … } static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, struct page *page, nid_t ino, enum page_type type, bool force) { … } void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) { … } void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, struct inode *inode, struct page *page, nid_t ino, enum page_type type) { … } void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) { … } /* * Fill the locked page with data located in the block address. * A caller needs to unlock the page on failure. */ int f2fs_submit_page_bio(struct f2fs_io_info *fio) { … } static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, block_t last_blkaddr, block_t cur_blkaddr) { … } static bool io_type_is_mergeable(struct f2fs_bio_info *io, struct f2fs_io_info *fio) { … } static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, struct f2fs_bio_info *io, struct f2fs_io_info *fio, block_t last_blkaddr, block_t cur_blkaddr) { … } static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, struct page *page, enum temp_type temp) { … } static void del_bio_entry(struct bio_entry *be) { … } static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, struct page *page) { … } void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, struct bio **bio, struct page *page) { … } int f2fs_merge_page_bio(struct f2fs_io_info *fio) { … } #ifdef CONFIG_BLK_DEV_ZONED static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) { … } #endif void f2fs_submit_page_write(struct f2fs_io_info *fio) { … } static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, unsigned nr_pages, blk_opf_t op_flag, pgoff_t first_idx, bool for_write) { … } /* This can handle encryption stuffs */ static int f2fs_submit_page_read(struct inode *inode, struct page *page, block_t blkaddr, blk_opf_t op_flags, bool for_write) { … } static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { … } /* * Lock ordering for the change of data block address: * ->data_page * ->node_page * update block addresses in the node page */ void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { … } void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) { … } /* dn->ofs_in_node will be returned with up-to-date last block pointer */ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) { … } /* Should keep dn->ofs_in_node unchanged */ int f2fs_reserve_new_block(struct dnode_of_data *dn) { … } int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) { … } struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs) { … } struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index, pgoff_t *next_pgofs) { … } /* * If it tries to access a hole, return an error. * Because, the callers, functions in dir.c and GC, should be able to know * whether this page exists or not. */ struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index, bool for_write) { … } /* * Caller ensures that this data page is never allocated. * A new zero-filled data page is allocated in the page cache. * * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and * f2fs_unlock_op(). * Note that, ipage is set only by make_empty_dir, and if any error occur, * ipage should be released by this function. */ struct page *f2fs_get_new_data_page(struct inode *inode, struct page *ipage, pgoff_t index, bool new_i_size) { … } static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) { … } static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) { … } static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) { … } int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) { … } static int f2fs_map_no_dnode(struct inode *inode, struct f2fs_map_blocks *map, struct dnode_of_data *dn, pgoff_t pgoff) { … } static bool f2fs_map_blocks_cached(struct inode *inode, struct f2fs_map_blocks *map, int flag) { … } static bool map_is_mergeable(struct f2fs_sb_info *sbi, struct f2fs_map_blocks *map, block_t blkaddr, int flag, int bidx, int ofs) { … } /* * f2fs_map_blocks() tries to find or build mapping relationship which * maps continuous logical blocks to physical blocks, and return such * info via f2fs_map_blocks structure. */ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) { … } bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) { … } static inline u64 bytes_to_blks(struct inode *inode, u64 bytes) { … } static inline u64 blks_to_bytes(struct inode *inode, u64 blks) { … } static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { … } static loff_t max_inode_blocks(struct inode *inode) { … } int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { … } static inline loff_t f2fs_readpage_limit(struct inode *inode) { … } static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) { … } static int f2fs_read_single_page(struct inode *inode, struct folio *folio, unsigned nr_pages, struct f2fs_map_blocks *map, struct bio **bio_ret, sector_t *last_block_in_bio, struct readahead_control *rac) { … } #ifdef CONFIG_F2FS_FS_COMPRESSION int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, unsigned nr_pages, sector_t *last_block_in_bio, struct readahead_control *rac, bool for_write) { … } #endif /* * This function was originally taken from fs/mpage.c, and customized for f2fs. * Major change was from block_size == page_size in f2fs by default. */ static int f2fs_mpage_readpages(struct inode *inode, struct readahead_control *rac, struct folio *folio) { … } static int f2fs_read_data_folio(struct file *file, struct folio *folio) { … } static void f2fs_readahead(struct readahead_control *rac) { … } int f2fs_encrypt_one_page(struct f2fs_io_info *fio) { … } static inline bool check_inplace_update_policy(struct inode *inode, struct f2fs_io_info *fio) { … } bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) { … } bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) { … } static inline bool need_inplace_update(struct f2fs_io_info *fio) { … } int f2fs_do_write_data_page(struct f2fs_io_info *fio) { … } int f2fs_write_single_data_page(struct page *page, int *submitted, struct bio **bio, sector_t *last_block, struct writeback_control *wbc, enum iostat_type io_type, int compr_blocks, bool allow_balance) { … } static int f2fs_write_data_page(struct page *page, struct writeback_control *wbc) { … } /* * This function was copied from write_cache_pages from mm/page-writeback.c. * The major change is making write step of cold data page separately from * warm/hot data page. */ static int f2fs_write_cache_pages(struct address_space *mapping, struct writeback_control *wbc, enum iostat_type io_type) { … } static inline bool __should_serialize_io(struct inode *inode, struct writeback_control *wbc) { … } static int __f2fs_write_data_pages(struct address_space *mapping, struct writeback_control *wbc, enum iostat_type io_type) { … } static int f2fs_write_data_pages(struct address_space *mapping, struct writeback_control *wbc) { … } void f2fs_write_failed(struct inode *inode, loff_t to) { … } static int prepare_write_begin(struct f2fs_sb_info *sbi, struct page *page, loff_t pos, unsigned len, block_t *blk_addr, bool *node_changed) { … } static int __find_data_block(struct inode *inode, pgoff_t index, block_t *blk_addr) { … } static int __reserve_data_block(struct inode *inode, pgoff_t index, block_t *blk_addr, bool *node_changed) { … } static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, struct page *page, loff_t pos, unsigned int len, block_t *blk_addr, bool *node_changed, bool *use_cow) { … } static int f2fs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, struct page **pagep, void **fsdata) { … } static int f2fs_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { … } void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) { … } bool f2fs_release_folio(struct folio *folio, gfp_t wait) { … } static bool f2fs_dirty_data_folio(struct address_space *mapping, struct folio *folio) { … } static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) { … } static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) { … } #ifdef CONFIG_SWAP static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, unsigned int blkcnt) { … } static int check_swap_activate(struct swap_info_struct *sis, struct file *swap_file, sector_t *span) { … } static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { … } static void f2fs_swap_deactivate(struct file *file) { … } #else static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { return -EOPNOTSUPP; } static void f2fs_swap_deactivate(struct file *file) { } #endif const struct address_space_operations f2fs_dblock_aops = …; void f2fs_clear_page_cache_dirty_tag(struct page *page) { … } int __init f2fs_init_post_read_processing(void) { … } void f2fs_destroy_post_read_processing(void) { … } int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) { … } void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) { … } int __init f2fs_init_bio_entry_cache(void) { … } void f2fs_destroy_bio_entry_cache(void) { … } static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { … } const struct iomap_ops f2fs_iomap_ops = …;
Codebase Browser
linux