linux/fs/bcachefs/alloc_background.c

// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "backpointers.h"
#include "bkey_buf.h"
#include "btree_cache.h"
#include "btree_io.h"
#include "btree_key_cache.h"
#include "btree_update.h"
#include "btree_update_interior.h"
#include "btree_gc.h"
#include "btree_write_buffer.h"
#include "buckets.h"
#include "buckets_waiting_for_journal.h"
#include "clock.h"
#include "debug.h"
#include "disk_accounting.h"
#include "ec.h"
#include "error.h"
#include "lru.h"
#include "recovery.h"
#include "trace.h"
#include "varint.h"

#include <linux/kthread.h>
#include <linux/math64.h>
#include <linux/random.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/sched/task.h>
#include <linux/sort.h>

static void bch2_discard_one_bucket_fast(struct bch_dev *, u64);

/* Persistent alloc info: */

static const unsigned BCH_ALLOC_V1_FIELD_BYTES[] = …;

struct bkey_alloc_unpacked { … };

static inline u64 alloc_field_v1_get(const struct bch_alloc *a,
				     const void **p, unsigned field)
{ … }

static void bch2_alloc_unpack_v1(struct bkey_alloc_unpacked *out,
				 struct bkey_s_c k)
{ … }

static int bch2_alloc_unpack_v2(struct bkey_alloc_unpacked *out,
				struct bkey_s_c k)
{ … }

static int bch2_alloc_unpack_v3(struct bkey_alloc_unpacked *out,
				struct bkey_s_c k)
{ … }

static struct bkey_alloc_unpacked bch2_alloc_unpack(struct bkey_s_c k)
{ … }

static unsigned bch_alloc_v1_val_u64s(const struct bch_alloc *a)
{ … }

int bch2_alloc_v1_invalid(struct bch_fs *c, struct bkey_s_c k,
			  enum bch_validate_flags flags,
			  struct printbuf *err)
{ … }

int bch2_alloc_v2_invalid(struct bch_fs *c, struct bkey_s_c k,
			  enum bch_validate_flags flags,
			  struct printbuf *err)
{ … }

int bch2_alloc_v3_invalid(struct bch_fs *c, struct bkey_s_c k,
			  enum bch_validate_flags flags,
			  struct printbuf *err)
{ … }

int bch2_alloc_v4_invalid(struct bch_fs *c, struct bkey_s_c k,
			  enum bch_validate_flags flags, struct printbuf *err)
{ … }

void bch2_alloc_v4_swab(struct bkey_s k)
{ … }

void bch2_alloc_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
{ … }

void __bch2_alloc_to_v4(struct bkey_s_c k, struct bch_alloc_v4 *out)
{ … }

static noinline struct bkey_i_alloc_v4 *
__bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k)
{ … }

static inline struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut_inlined(struct btree_trans *trans, struct bkey_s_c k)
{ … }

struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k)
{ … }

struct bkey_i_alloc_v4 *
bch2_trans_start_alloc_update_noupdate(struct btree_trans *trans, struct btree_iter *iter,
				       struct bpos pos)
{ … }

__flatten
struct bkey_i_alloc_v4 *bch2_trans_start_alloc_update(struct btree_trans *trans, struct bpos pos,
						      enum btree_iter_update_trigger_flags flags)
{ … }

static struct bpos alloc_gens_pos(struct bpos pos, unsigned *offset)
{ … }

static struct bpos bucket_gens_pos_to_alloc(struct bpos pos, unsigned offset)
{ … }

static unsigned alloc_gen(struct bkey_s_c k, unsigned offset)
{ … }

int bch2_bucket_gens_invalid(struct bch_fs *c, struct bkey_s_c k,
			     enum bch_validate_flags flags,
			     struct printbuf *err)
{ … }

void bch2_bucket_gens_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
{ … }

int bch2_bucket_gens_init(struct bch_fs *c)
{ … }

int bch2_alloc_read(struct bch_fs *c)
{ … }

/* Free space/discard btree: */

static int bch2_bucket_do_index(struct btree_trans *trans,
				struct bch_dev *ca,
				struct bkey_s_c alloc_k,
				const struct bch_alloc_v4 *a,
				bool set)
{ … }

static noinline int bch2_bucket_gen_update(struct btree_trans *trans,
					   struct bpos bucket, u8 gen)
{ … }

static inline int bch2_dev_data_type_accounting_mod(struct btree_trans *trans, struct bch_dev *ca,
						    enum bch_data_type data_type,
						    s64 delta_buckets,
						    s64 delta_sectors,
						    s64 delta_fragmented, unsigned flags)
{ … }

int bch2_alloc_key_to_dev_counters(struct btree_trans *trans, struct bch_dev *ca,
				   const struct bch_alloc_v4 *old,
				   const struct bch_alloc_v4 *new,
				   unsigned flags)
{ … }

int bch2_trigger_alloc(struct btree_trans *trans,
		       enum btree_id btree, unsigned level,
		       struct bkey_s_c old, struct bkey_s new,
		       enum btree_iter_update_trigger_flags flags)
{ … }

/*
 * This synthesizes deleted extents for holes, similar to BTREE_ITER_slots for
 * extents style btrees, but works on non-extents btrees:
 */
static struct bkey_s_c bch2_get_key_or_hole(struct btree_iter *iter, struct bpos end, struct bkey *hole)
{ … }

static bool next_bucket(struct bch_fs *c, struct bch_dev **ca, struct bpos *bucket)
{ … }

static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter,
					struct bch_dev **ca, struct bkey *hole)
{ … }

static noinline_for_stack
int bch2_check_alloc_key(struct btree_trans *trans,
			 struct bkey_s_c alloc_k,
			 struct btree_iter *alloc_iter,
			 struct btree_iter *discard_iter,
			 struct btree_iter *freespace_iter,
			 struct btree_iter *bucket_gens_iter)
{ … }

static noinline_for_stack
int bch2_check_alloc_hole_freespace(struct btree_trans *trans,
				    struct bch_dev *ca,
				    struct bpos start,
				    struct bpos *end,
				    struct btree_iter *freespace_iter)
{ … }

static noinline_for_stack
int bch2_check_alloc_hole_bucket_gens(struct btree_trans *trans,
				      struct bpos start,
				      struct bpos *end,
				      struct btree_iter *bucket_gens_iter)
{ … }

static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_trans *trans,
					      struct btree_iter *iter)
{ … }

/*
 * We've already checked that generation numbers in the bucket_gens btree are
 * valid for buckets that exist; this just checks for keys for nonexistent
 * buckets.
 */
static noinline_for_stack
int bch2_check_bucket_gens_key(struct btree_trans *trans,
			       struct btree_iter *iter,
			       struct bkey_s_c k)
{ … }

int bch2_check_alloc_info(struct bch_fs *c)
{ … }

static int bch2_check_alloc_to_lru_ref(struct btree_trans *trans,
				       struct btree_iter *alloc_iter,
				       struct bkey_buf *last_flushed)
{ … }

int bch2_check_alloc_to_lru_refs(struct bch_fs *c)
{ … }

static int discard_in_flight_add(struct bch_dev *ca, u64 bucket, bool in_progress)
{ … }

static void discard_in_flight_remove(struct bch_dev *ca, u64 bucket)
{ … }

struct discard_buckets_state { … };

static int bch2_discard_one_bucket(struct btree_trans *trans,
				   struct bch_dev *ca,
				   struct btree_iter *need_discard_iter,
				   struct bpos *discard_pos_done,
				   struct discard_buckets_state *s)
{ … }

static void bch2_do_discards_work(struct work_struct *work)
{ … }

void bch2_dev_do_discards(struct bch_dev *ca)
{ … }

void bch2_do_discards(struct bch_fs *c)
{ … }

static int bch2_clear_bucket_needs_discard(struct btree_trans *trans, struct bpos bucket)
{ … }

static void bch2_do_discards_fast_work(struct work_struct *work)
{ … }

static void bch2_discard_one_bucket_fast(struct bch_dev *ca, u64 bucket)
{ … }

static int invalidate_one_bucket(struct btree_trans *trans,
				 struct btree_iter *lru_iter,
				 struct bkey_s_c lru_k,
				 s64 *nr_to_invalidate)
{ … }

static struct bkey_s_c next_lru_key(struct btree_trans *trans, struct btree_iter *iter,
				    struct bch_dev *ca, bool *wrapped)
{ … }

static void bch2_do_invalidates_work(struct work_struct *work)
{ … }

void bch2_dev_do_invalidates(struct bch_dev *ca)
{ … }

void bch2_do_invalidates(struct bch_fs *c)
{ … }

int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca,
			    u64 bucket_start, u64 bucket_end)
{ … }

int bch2_fs_freespace_init(struct bch_fs *c)
{ … }

/* Bucket IO clocks: */

int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev,
			      size_t bucket_nr, int rw)
{ … }

/* Startup/shutdown (ro/rw): */

void bch2_recalc_capacity(struct bch_fs *c)
{ … }

u64 bch2_min_rw_member_capacity(struct bch_fs *c)
{ … }

static bool bch2_dev_has_open_write_point(struct bch_fs *c, struct bch_dev *ca)
{ … }

/* device goes ro: */
void bch2_dev_allocator_remove(struct bch_fs *c, struct bch_dev *ca)
{ … }

/* device goes rw: */
void bch2_dev_allocator_add(struct bch_fs *c, struct bch_dev *ca)
{ … }

void bch2_dev_allocator_background_exit(struct bch_dev *ca)
{ … }

void bch2_dev_allocator_background_init(struct bch_dev *ca)
{ … }

void bch2_fs_allocator_background_init(struct bch_fs *c)
{ … }