#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/sched/mm.h>
#include <linux/prefetch.h>
#include <linux/kthread.h>
#include <linux/swap.h>
#include <linux/timer.h>
#include <linux/freezer.h>
#include <linux/sched/signal.h>
#include <linux/random.h>
#include "f2fs.h"
#include "segment.h"
#include "node.h"
#include "gc.h"
#include "iostat.h"
#include <trace/events/f2fs.h>
#define __reverse_ffz(x) …
static struct kmem_cache *discard_entry_slab;
static struct kmem_cache *discard_cmd_slab;
static struct kmem_cache *sit_entry_set_slab;
static struct kmem_cache *revoke_entry_slab;
static unsigned long __reverse_ulong(unsigned char *str)
{ … }
static inline unsigned long __reverse_ffs(unsigned long word)
{ … }
static unsigned long __find_rev_next_bit(const unsigned long *addr,
unsigned long size, unsigned long offset)
{ … }
static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
unsigned long size, unsigned long offset)
{ … }
bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
{ … }
void f2fs_abort_atomic_write(struct inode *inode, bool clean)
{ … }
static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
block_t new_addr, block_t *old_addr, bool recover)
{ … }
static void __complete_revoke_list(struct inode *inode, struct list_head *head,
bool revoke)
{ … }
static int __f2fs_commit_atomic_write(struct inode *inode)
{ … }
int f2fs_commit_atomic_write(struct inode *inode)
{ … }
void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
{ … }
static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
{ … }
void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
{ … }
static int __submit_flush_wait(struct f2fs_sb_info *sbi,
struct block_device *bdev)
{ … }
static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
{ … }
static int issue_flush_thread(void *data)
{ … }
int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
{ … }
int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
{ … }
void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
{ … }
int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
{ … }
static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
enum dirty_type dirty_type)
{ … }
static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
enum dirty_type dirty_type)
{ … }
static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
{ … }
void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
{ … }
block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
{ … }
int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
{ … }
static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
{ … }
static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart,
block_t start, block_t len)
{ … }
static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
{ … }
static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
block_t blkaddr)
{ … }
static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
block_t blkaddr,
struct discard_cmd **prev_entry,
struct discard_cmd **next_entry,
struct rb_node ***insert_p,
struct rb_node **insert_parent)
{ … }
static void __detach_discard_cmd(struct discard_cmd_control *dcc,
struct discard_cmd *dc)
{ … }
static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_cmd *dc)
{ … }
static void f2fs_submit_discard_endio(struct bio *bio)
{ … }
static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
block_t start, block_t end)
{ … }
static void __init_discard_policy(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy,
int discard_type, unsigned int granularity)
{ … }
static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart,
block_t start, block_t len);
#ifdef CONFIG_BLK_DEV_ZONED
static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
struct discard_cmd *dc, blk_opf_t flag,
struct list_head *wait_list,
unsigned int *issued)
{ … }
#endif
static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy,
struct discard_cmd *dc, int *issued)
{ … }
static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart,
block_t start, block_t len)
{ … }
static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
struct discard_cmd *dc)
{ … }
static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_cmd *dc, block_t blkaddr)
{ … }
static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart,
block_t start, block_t len)
{ … }
#ifdef CONFIG_BLK_DEV_ZONED
static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t lblkstart,
block_t blklen)
{ … }
#endif
static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t blklen)
{ … }
static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy, int *issued)
{ … }
static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy);
static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy)
{ … }
static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
{ … }
void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
{ … }
static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
struct discard_cmd *dc)
{ … }
static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy,
block_t start, block_t end)
{ … }
static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy)
{ … }
static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
{ … }
void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
{ … }
bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
{ … }
static int issue_discard_thread(void *data)
{ … }
#ifdef CONFIG_BLK_DEV_ZONED
static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t blklen)
{ … }
#endif
static int __issue_discard_async(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t blklen)
{ … }
static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
block_t blkstart, block_t blklen)
{ … }
static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
bool check_only)
{ … }
static void release_discard_addr(struct discard_entry *entry)
{ … }
void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
{ … }
static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
{ … }
void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
struct cp_control *cpc)
{ … }
int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
{ … }
static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
{ … }
static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
{ … }
static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
{ … }
static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
unsigned int segno, int modified)
{ … }
static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
block_t blkaddr)
{ … }
static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
unsigned long long old_mtime)
{ … }
static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
{ … }
void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
{ … }
bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
{ … }
static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
{ … }
int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
{ … }
struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
{ … }
void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
void *src, block_t blk_addr)
{ … }
static void write_sum_page(struct f2fs_sb_info *sbi,
struct f2fs_summary_block *sum_blk, block_t blk_addr)
{ … }
static void write_current_sum_page(struct f2fs_sb_info *sbi,
int type, block_t blk_addr)
{ … }
static int is_next_segment_free(struct f2fs_sb_info *sbi,
struct curseg_info *curseg)
{ … }
static int get_new_segment(struct f2fs_sb_info *sbi,
unsigned int *newseg, bool new_sec, bool pinning)
{ … }
static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
{ … }
static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
{ … }
static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
{ … }
static int __next_free_blkoff(struct f2fs_sb_info *sbi,
int segno, block_t start)
{ … }
static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
struct curseg_info *seg)
{ … }
bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
{ … }
static int change_curseg(struct f2fs_sb_info *sbi, int type)
{ … }
static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
int alloc_mode, unsigned long long age);
static int get_atssr_segment(struct f2fs_sb_info *sbi, int type,
int target_type, int alloc_mode,
unsigned long long age)
{ … }
static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force)
{ … }
int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
{ … }
int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi)
{ … }
static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
{ … }
void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
{ … }
static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
{ … }
void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
{ … }
static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
int alloc_mode, unsigned long long age)
{ … }
static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
{ … }
int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
unsigned int start, unsigned int end)
{ … }
static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
bool new_sec, bool force)
{ … }
int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
{ … }
int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi)
{ … }
int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
{ … }
bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
struct cp_control *cpc)
{ … }
static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
struct discard_policy *dpolicy,
unsigned int start, unsigned int end)
{ … }
int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
{ … }
int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint)
{ … }
enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
enum page_type type, enum temp_type temp)
{ … }
static int __get_segment_type_2(struct f2fs_io_info *fio)
{ … }
static int __get_segment_type_4(struct f2fs_io_info *fio)
{ … }
static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
{ … }
static int __get_segment_type_6(struct f2fs_io_info *fio)
{ … }
int f2fs_get_segment_temp(int seg_type)
{ … }
static int __get_segment_type(struct f2fs_io_info *fio)
{ … }
static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
struct curseg_info *seg)
{ … }
static void reset_curseg_fields(struct curseg_info *curseg)
{ … }
int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
block_t old_blkaddr, block_t *new_blkaddr,
struct f2fs_summary *sum, int type,
struct f2fs_io_info *fio)
{ … }
void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
block_t blkaddr, unsigned int blkcnt)
{ … }
static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
{ … }
void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
enum iostat_type io_type)
{ … }
void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
{ … }
void f2fs_outplace_write_data(struct dnode_of_data *dn,
struct f2fs_io_info *fio)
{ … }
int f2fs_inplace_write_data(struct f2fs_io_info *fio)
{ … }
static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
unsigned int segno)
{ … }
void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
block_t old_blkaddr, block_t new_blkaddr,
bool recover_curseg, bool recover_newaddr,
bool from_gc)
{ … }
void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
block_t old_addr, block_t new_addr,
unsigned char version, bool recover_curseg,
bool recover_newaddr)
{ … }
void f2fs_wait_on_page_writeback(struct page *page,
enum page_type type, bool ordered, bool locked)
{ … }
void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
{ … }
void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
block_t len)
{ … }
static int read_compacted_summaries(struct f2fs_sb_info *sbi)
{ … }
static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
{ … }
static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
{ … }
static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
{ … }
static void write_normal_summaries(struct f2fs_sb_info *sbi,
block_t blkaddr, int type)
{ … }
void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
{ … }
void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
{ … }
int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
unsigned int val, int alloc)
{ … }
static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
unsigned int segno)
{ … }
static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
unsigned int start)
{ … }
static struct sit_entry_set *grab_sit_entry_set(void)
{ … }
static void release_sit_entry_set(struct sit_entry_set *ses)
{ … }
static void adjust_sit_entry_set(struct sit_entry_set *ses,
struct list_head *head)
{ … }
static void add_sit_entry(unsigned int segno, struct list_head *head)
{ … }
static void add_sits_in_set(struct f2fs_sb_info *sbi)
{ … }
static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
{ … }
void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
{ … }
static int build_sit_info(struct f2fs_sb_info *sbi)
{ … }
static int build_free_segmap(struct f2fs_sb_info *sbi)
{ … }
static int build_curseg(struct f2fs_sb_info *sbi)
{ … }
static int build_sit_entries(struct f2fs_sb_info *sbi)
{ … }
static void init_free_segmap(struct f2fs_sb_info *sbi)
{ … }
static void init_dirty_segmap(struct f2fs_sb_info *sbi)
{ … }
static int init_victim_secmap(struct f2fs_sb_info *sbi)
{ … }
static int build_dirty_segmap(struct f2fs_sb_info *sbi)
{ … }
static int sanity_check_curseg(struct f2fs_sb_info *sbi)
{ … }
#ifdef CONFIG_BLK_DEV_ZONED
static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
struct f2fs_dev_info *fdev,
struct blk_zone *zone)
{ … }
static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
block_t zone_blkaddr)
{ … }
static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
void *data)
{ … }
static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
{ … }
int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
{ … }
struct check_zone_write_pointer_args { … };
static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
void *data)
{ … }
int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
{ … }
static inline unsigned int f2fs_usable_zone_blks_in_seg(
struct f2fs_sb_info *sbi, unsigned int segno)
{ … }
#else
int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
{
return 0;
}
int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
{
return 0;
}
static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
unsigned int segno)
{
return 0;
}
#endif
unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
unsigned int segno)
{ … }
unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
unsigned int segno)
{ … }
static void init_min_max_mtime(struct f2fs_sb_info *sbi)
{ … }
int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
{ … }
static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
enum dirty_type dirty_type)
{ … }
static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
{ … }
static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
{ … }
static void destroy_curseg(struct f2fs_sb_info *sbi)
{ … }
static void destroy_free_segmap(struct f2fs_sb_info *sbi)
{ … }
static void destroy_sit_info(struct f2fs_sb_info *sbi)
{ … }
void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
{ … }
int __init f2fs_create_segment_manager_caches(void)
{ … }
void f2fs_destroy_segment_manager_caches(void)
{ … }