linux/fs/f2fs/segment.c

// SPDX-License-Identifier: GPL-2.0
/*
 * fs/f2fs/segment.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/sched/mm.h>
#include <linux/prefetch.h>
#include <linux/kthread.h>
#include <linux/swap.h>
#include <linux/timer.h>
#include <linux/freezer.h>
#include <linux/sched/signal.h>
#include <linux/random.h>

#include "f2fs.h"
#include "segment.h"
#include "node.h"
#include "gc.h"
#include "iostat.h"
#include <trace/events/f2fs.h>

#define __reverse_ffz(x)

static struct kmem_cache *discard_entry_slab;
static struct kmem_cache *discard_cmd_slab;
static struct kmem_cache *sit_entry_set_slab;
static struct kmem_cache *revoke_entry_slab;

static unsigned long __reverse_ulong(unsigned char *str)
{}

/*
 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
 * MSB and LSB are reversed in a byte by f2fs_set_bit.
 */
static inline unsigned long __reverse_ffs(unsigned long word)
{}

/*
 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
 * f2fs_set_bit makes MSB and LSB reversed in a byte.
 * @size must be integral times of unsigned long.
 * Example:
 *                             MSB <--> LSB
 *   f2fs_set_bit(0, bitmap) => 1000 0000
 *   f2fs_set_bit(7, bitmap) => 0000 0001
 */
static unsigned long __find_rev_next_bit(const unsigned long *addr,
			unsigned long size, unsigned long offset)
{}

static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
			unsigned long size, unsigned long offset)
{}

bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
{}

void f2fs_abort_atomic_write(struct inode *inode, bool clean)
{}

static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
			block_t new_addr, block_t *old_addr, bool recover)
{}

static void __complete_revoke_list(struct inode *inode, struct list_head *head,
					bool revoke)
{}

static int __f2fs_commit_atomic_write(struct inode *inode)
{}

int f2fs_commit_atomic_write(struct inode *inode)
{}

/*
 * This function balances dirty node and dentry pages.
 * In addition, it controls garbage collection.
 */
void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
{}

static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
{}

void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
{}

static int __submit_flush_wait(struct f2fs_sb_info *sbi,
				struct block_device *bdev)
{}

static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
{}

static int issue_flush_thread(void *data)
{}

int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
{}

int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
{}

void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
{}

int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
{}

static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
		enum dirty_type dirty_type)
{}

static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
		enum dirty_type dirty_type)
{}

/*
 * Should not occur error such as -ENOMEM.
 * Adding dirty entry into seglist is not critical operation.
 * If a given segment is one of current working segments, it won't be added.
 */
static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
{}

/* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
{}

block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
{}

int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
{}

/* This is only used by SBI_CP_DISABLED */
static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
{}

static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
		struct block_device *bdev, block_t lstart,
		block_t start, block_t len)
{}

static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
{}

static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
						block_t blkaddr)
{}

static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
				block_t blkaddr,
				struct discard_cmd **prev_entry,
				struct discard_cmd **next_entry,
				struct rb_node ***insert_p,
				struct rb_node **insert_parent)
{}

static void __detach_discard_cmd(struct discard_cmd_control *dcc,
							struct discard_cmd *dc)
{}

static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
							struct discard_cmd *dc)
{}

static void f2fs_submit_discard_endio(struct bio *bio)
{}

static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
				block_t start, block_t end)
{}

static void __init_discard_policy(struct f2fs_sb_info *sbi,
				struct discard_policy *dpolicy,
				int discard_type, unsigned int granularity)
{}

static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
				struct block_device *bdev, block_t lstart,
				block_t start, block_t len);

#ifdef CONFIG_BLK_DEV_ZONED
static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
				   struct discard_cmd *dc, blk_opf_t flag,
				   struct list_head *wait_list,
				   unsigned int *issued)
{}
#endif

/* this function is copied from blkdev_issue_discard from block/blk-lib.c */
static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
				struct discard_policy *dpolicy,
				struct discard_cmd *dc, int *issued)
{}

static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
				struct block_device *bdev, block_t lstart,
				block_t start, block_t len)
{}

static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
						struct discard_cmd *dc)
{}

static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
				struct discard_cmd *dc, block_t blkaddr)
{}

static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
				struct block_device *bdev, block_t lstart,
				block_t start, block_t len)
{}

#ifdef CONFIG_BLK_DEV_ZONED
static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,
		struct block_device *bdev, block_t blkstart, block_t lblkstart,
		block_t blklen)
{}
#endif

static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
		struct block_device *bdev, block_t blkstart, block_t blklen)
{}

static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
		struct discard_policy *dpolicy, int *issued)
{}
static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
					struct discard_policy *dpolicy);

static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
					struct discard_policy *dpolicy)
{}

static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
{}

void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
{}

static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
							struct discard_cmd *dc)
{}

static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
						struct discard_policy *dpolicy,
						block_t start, block_t end)
{}

static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
						struct discard_policy *dpolicy)
{}

/* This should be covered by global mutex, &sit_i->sentry_lock */
static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
{}

void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
{}

/**
 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
 * @sbi: the f2fs_sb_info data for discard cmd to issue
 *
 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
 *
 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
 */
bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
{}

static int issue_discard_thread(void *data)
{}

#ifdef CONFIG_BLK_DEV_ZONED
static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
		struct block_device *bdev, block_t blkstart, block_t blklen)
{}
#endif

static int __issue_discard_async(struct f2fs_sb_info *sbi,
		struct block_device *bdev, block_t blkstart, block_t blklen)
{}

static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
				block_t blkstart, block_t blklen)
{}

static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
							bool check_only)
{}

static void release_discard_addr(struct discard_entry *entry)
{}

void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
{}

/*
 * Should call f2fs_clear_prefree_segments after checkpoint is done.
 */
static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
{}

void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
						struct cp_control *cpc)
{}

int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
{}

static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
{}

static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
{}

static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
{}

static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
					unsigned int segno, int modified)
{}

static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
								block_t blkaddr)
{}

static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
						unsigned long long old_mtime)
{}

static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
{}

void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
{}

bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
{}

static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
{}

/*
 * Calculate the number of current summary pages for writing
 */
int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
{}

/*
 * Caller should put this summary page
 */
struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
{}

void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
					void *src, block_t blk_addr)
{}

static void write_sum_page(struct f2fs_sb_info *sbi,
			struct f2fs_summary_block *sum_blk, block_t blk_addr)
{}

static void write_current_sum_page(struct f2fs_sb_info *sbi,
						int type, block_t blk_addr)
{}

static int is_next_segment_free(struct f2fs_sb_info *sbi,
				struct curseg_info *curseg)
{}

/*
 * Find a new segment from the free segments bitmap to right order
 * This function should be returned with success, otherwise BUG
 */
static int get_new_segment(struct f2fs_sb_info *sbi,
			unsigned int *newseg, bool new_sec, bool pinning)
{}

static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
{}

static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
{}

/*
 * Allocate a current working segment.
 * This function always allocates a free segment in LFS manner.
 */
static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
{}

static int __next_free_blkoff(struct f2fs_sb_info *sbi,
					int segno, block_t start)
{}

static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
		struct curseg_info *seg)
{}

bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
{}

/*
 * This function always allocates a used segment(from dirty seglist) by SSR
 * manner, so it should recover the existing segment information of valid blocks
 */
static int change_curseg(struct f2fs_sb_info *sbi, int type)
{}

static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
				int alloc_mode, unsigned long long age);

static int get_atssr_segment(struct f2fs_sb_info *sbi, int type,
					int target_type, int alloc_mode,
					unsigned long long age)
{}

static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force)
{}

int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
{}

int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi)
{}

static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
{}

void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
{}

static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
{}

void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
{}

static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
				int alloc_mode, unsigned long long age)
{}

static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
{}

int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
					unsigned int start, unsigned int end)
{}

static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
						bool new_sec, bool force)
{}

int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
{}

int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi)
{}

int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
{}

bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
						struct cp_control *cpc)
{}

static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
					struct discard_policy *dpolicy,
					unsigned int start, unsigned int end)
{}

int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
{}

int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint)
{}

/*
 * This returns write hints for each segment type. This hints will be
 * passed down to block layer as below by default.
 *
 * User                  F2FS                     Block
 * ----                  ----                     -----
 *                       META                     WRITE_LIFE_NONE|REQ_META
 *                       HOT_NODE                 WRITE_LIFE_NONE
 *                       WARM_NODE                WRITE_LIFE_MEDIUM
 *                       COLD_NODE                WRITE_LIFE_LONG
 * ioctl(COLD)           COLD_DATA                WRITE_LIFE_EXTREME
 * extension list        "                        "
 *
 * -- buffered io
 *                       COLD_DATA                WRITE_LIFE_EXTREME
 *                       HOT_DATA                 WRITE_LIFE_SHORT
 *                       WARM_DATA                WRITE_LIFE_NOT_SET
 *
 * -- direct io
 * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
 * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
 * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_NOT_SET
 * WRITE_LIFE_NONE       "                        WRITE_LIFE_NONE
 * WRITE_LIFE_MEDIUM     "                        WRITE_LIFE_MEDIUM
 * WRITE_LIFE_LONG       "                        WRITE_LIFE_LONG
 */
enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
				enum page_type type, enum temp_type temp)
{}

static int __get_segment_type_2(struct f2fs_io_info *fio)
{}

static int __get_segment_type_4(struct f2fs_io_info *fio)
{}

static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
{}

static int __get_segment_type_6(struct f2fs_io_info *fio)
{}

int f2fs_get_segment_temp(int seg_type)
{}

static int __get_segment_type(struct f2fs_io_info *fio)
{}

static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
		struct curseg_info *seg)
{}

static void reset_curseg_fields(struct curseg_info *curseg)
{}

int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
		block_t old_blkaddr, block_t *new_blkaddr,
		struct f2fs_summary *sum, int type,
		struct f2fs_io_info *fio)
{}

void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
					block_t blkaddr, unsigned int blkcnt)
{}

static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
{}

void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
					enum iostat_type io_type)
{}

void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
{}

void f2fs_outplace_write_data(struct dnode_of_data *dn,
					struct f2fs_io_info *fio)
{}

int f2fs_inplace_write_data(struct f2fs_io_info *fio)
{}

static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
						unsigned int segno)
{}

void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
				block_t old_blkaddr, block_t new_blkaddr,
				bool recover_curseg, bool recover_newaddr,
				bool from_gc)
{}

void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
				block_t old_addr, block_t new_addr,
				unsigned char version, bool recover_curseg,
				bool recover_newaddr)
{}

void f2fs_wait_on_page_writeback(struct page *page,
				enum page_type type, bool ordered, bool locked)
{}

void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
{}

void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
								block_t len)
{}

static int read_compacted_summaries(struct f2fs_sb_info *sbi)
{}

static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
{}

static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
{}

static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
{}

static void write_normal_summaries(struct f2fs_sb_info *sbi,
					block_t blkaddr, int type)
{}

void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
{}

void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
{}

int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
					unsigned int val, int alloc)
{}

static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
					unsigned int segno)
{}

static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
					unsigned int start)
{}

static struct sit_entry_set *grab_sit_entry_set(void)
{}

static void release_sit_entry_set(struct sit_entry_set *ses)
{}

static void adjust_sit_entry_set(struct sit_entry_set *ses,
						struct list_head *head)
{}

static void add_sit_entry(unsigned int segno, struct list_head *head)
{}

static void add_sits_in_set(struct f2fs_sb_info *sbi)
{}

static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
{}

/*
 * CP calls this function, which flushes SIT entries including sit_journal,
 * and moves prefree segs to free segs.
 */
void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
{}

static int build_sit_info(struct f2fs_sb_info *sbi)
{}

static int build_free_segmap(struct f2fs_sb_info *sbi)
{}

static int build_curseg(struct f2fs_sb_info *sbi)
{}

static int build_sit_entries(struct f2fs_sb_info *sbi)
{}

static void init_free_segmap(struct f2fs_sb_info *sbi)
{}

static void init_dirty_segmap(struct f2fs_sb_info *sbi)
{}

static int init_victim_secmap(struct f2fs_sb_info *sbi)
{}

static int build_dirty_segmap(struct f2fs_sb_info *sbi)
{}

static int sanity_check_curseg(struct f2fs_sb_info *sbi)
{}

#ifdef CONFIG_BLK_DEV_ZONED
static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
				    struct f2fs_dev_info *fdev,
				    struct blk_zone *zone)
{}

static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
						  block_t zone_blkaddr)
{}

static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
			      void *data)
{}

static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
{}

int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
{}

struct check_zone_write_pointer_args {};

static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
				      void *data)
{}

int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
{}

/*
 * Return the number of usable blocks in a segment. The number of blocks
 * returned is always equal to the number of blocks in a segment for
 * segments fully contained within a sequential zone capacity or a
 * conventional zone. For segments partially contained in a sequential
 * zone capacity, the number of usable blocks up to the zone capacity
 * is returned. 0 is returned in all other cases.
 */
static inline unsigned int f2fs_usable_zone_blks_in_seg(
			struct f2fs_sb_info *sbi, unsigned int segno)
{}
#else
int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
{
	return 0;
}

int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
{
	return 0;
}

static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
							unsigned int segno)
{
	return 0;
}

#endif
unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
					unsigned int segno)
{}

unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
					unsigned int segno)
{}

/*
 * Update min, max modified time for cost-benefit GC algorithm
 */
static void init_min_max_mtime(struct f2fs_sb_info *sbi)
{}

int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
{}

static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
		enum dirty_type dirty_type)
{}

static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
{}

static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
{}

static void destroy_curseg(struct f2fs_sb_info *sbi)
{}

static void destroy_free_segmap(struct f2fs_sb_info *sbi)
{}

static void destroy_sit_info(struct f2fs_sb_info *sbi)
{}

void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
{}

int __init f2fs_create_segment_manager_caches(void)
{}

void f2fs_destroy_segment_manager_caches(void)
{}