// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bcachefs_ioctl.h"
#include "btree_cache.h"
#include "btree_journal_iter.h"
#include "btree_update.h"
#include "btree_write_buffer.h"
#include "buckets.h"
#include "compress.h"
#include "disk_accounting.h"
#include "error.h"
#include "journal_io.h"
#include "replicas.h"
/*
* Notes on disk accounting:
*
* We have two parallel sets of counters to be concerned with, and both must be
* kept in sync.
*
* - Persistent/on disk accounting, stored in the accounting btree and updated
* via btree write buffer updates that treat new accounting keys as deltas to
* apply to existing values. But reading from a write buffer btree is
* expensive, so we also have
*
* - In memory accounting, where accounting is stored as an array of percpu
* counters, indexed by an eytzinger array of disk acounting keys/bpos (which
* are the same thing, excepting byte swabbing on big endian).
*
* Cheap to read, but non persistent.
*
* Disk accounting updates are generated by transactional triggers; these run as
* keys enter and leave the btree, and can compare old and new versions of keys;
* the output of these triggers are deltas to the various counters.
*
* Disk accounting updates are done as btree write buffer updates, where the
* counters in the disk accounting key are deltas that will be applied to the
* counter in the btree when the key is flushed by the write buffer (or journal
* replay).
*
* To do a disk accounting update:
* - initialize a disk_accounting_pos, to specify which counter is being update
* - initialize counter deltas, as an array of 1-3 s64s
* - call bch2_disk_accounting_mod()
*
* This queues up the accounting update to be done at transaction commit time.
* Underneath, it's a normal btree write buffer update.
*
* The transaction commit path is responsible for propagating updates to the in
* memory counters, with bch2_accounting_mem_mod().
*
* The commit path also assigns every disk accounting update a unique version
* number, based on the journal sequence number and offset within that journal
* buffer; this is used by journal replay to determine which updates have been
* done.
*
* The transaction commit path also ensures that replicas entry accounting
* updates are properly marked in the superblock (so that we know whether we can
* mount without data being unavailable); it will update the superblock if
* bch2_accounting_mem_mod() tells it to.
*/
static const char * const disk_accounting_type_strs[] = {
#define x(t, n, ...) [n] = #t,
BCH_DISK_ACCOUNTING_TYPES()
#undef x
NULL
};
static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos,
s64 *d, unsigned nr)
{
struct bkey_i_accounting *acc = bkey_accounting_init(k);
acc->k.p = disk_accounting_pos_to_bpos(pos);
set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr);
memcpy_u64s_small(acc->v.d, d, nr);
}
int bch2_disk_accounting_mod(struct btree_trans *trans,
struct disk_accounting_pos *k,
s64 *d, unsigned nr, bool gc)
{
/* Normalize: */
switch (k->type) {
case BCH_DISK_ACCOUNTING_replicas:
bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp);
break;
}
BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS);
struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
accounting_key_init(&k_i.k, k, d, nr);
return likely(!gc)
? bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k)
: bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);
}
int bch2_mod_dev_cached_sectors(struct btree_trans *trans,
unsigned dev, s64 sectors,
bool gc)
{
struct disk_accounting_pos acc = {
.type = BCH_DISK_ACCOUNTING_replicas,
};
bch2_replicas_entry_cached(&acc.replicas, dev);
return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);
}
static inline bool is_zero(char *start, char *end)
{
BUG_ON(start > end);
for (; start < end; start++)
if (*start)
return false;
return true;
}
#define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member))
int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k,
enum bch_validate_flags flags)
{
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, k.k->p);
void *end = &acc_k + 1;
int ret = 0;
bkey_fsck_err_on(bversion_zero(k.k->bversion),
c, accounting_key_version_0,
"accounting key with version=0");
switch (acc_k.type) {
case BCH_DISK_ACCOUNTING_nr_inodes:
end = field_end(acc_k, nr_inodes);
break;
case BCH_DISK_ACCOUNTING_persistent_reserved:
end = field_end(acc_k, persistent_reserved);
break;
case BCH_DISK_ACCOUNTING_replicas:
bkey_fsck_err_on(!acc_k.replicas.nr_devs,
c, accounting_key_replicas_nr_devs_0,
"accounting key replicas entry with nr_devs=0");
bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs ||
(acc_k.replicas.nr_required > 1 &&
acc_k.replicas.nr_required == acc_k.replicas.nr_devs),
c, accounting_key_replicas_nr_required_bad,
"accounting key replicas entry with bad nr_required");
for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++)
bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1],
c, accounting_key_replicas_devs_unsorted,
"accounting key replicas entry with unsorted devs");
end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas);
break;
case BCH_DISK_ACCOUNTING_dev_data_type:
end = field_end(acc_k, dev_data_type);
break;
case BCH_DISK_ACCOUNTING_compression:
end = field_end(acc_k, compression);
break;
case BCH_DISK_ACCOUNTING_snapshot:
end = field_end(acc_k, snapshot);
break;
case BCH_DISK_ACCOUNTING_btree:
end = field_end(acc_k, btree);
break;
case BCH_DISK_ACCOUNTING_rebalance_work:
end = field_end(acc_k, rebalance_work);
break;
}
bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)),
c, accounting_key_junk_at_end,
"junk at end of accounting key");
fsck_err:
return ret;
}
void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k)
{
if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) {
prt_printf(out, "unknown type %u", k->type);
return;
}
prt_str(out, disk_accounting_type_strs[k->type]);
prt_str(out, " ");
switch (k->type) {
case BCH_DISK_ACCOUNTING_nr_inodes:
break;
case BCH_DISK_ACCOUNTING_persistent_reserved:
prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas);
break;
case BCH_DISK_ACCOUNTING_replicas:
bch2_replicas_entry_to_text(out, &k->replicas);
break;
case BCH_DISK_ACCOUNTING_dev_data_type:
prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev);
bch2_prt_data_type(out, k->dev_data_type.data_type);
break;
case BCH_DISK_ACCOUNTING_compression:
bch2_prt_compression_type(out, k->compression.type);
break;
case BCH_DISK_ACCOUNTING_snapshot:
prt_printf(out, "id=%u", k->snapshot.id);
break;
case BCH_DISK_ACCOUNTING_btree:
prt_printf(out, "btree=%s", bch2_btree_id_str(k->btree.id));
break;
}
}
void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
{
struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k);
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, k.k->p);
bch2_accounting_key_to_text(out, &acc_k);
for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++)
prt_printf(out, " %lli", acc.v->d[i]);
}
void bch2_accounting_swab(struct bkey_s k)
{
for (u64 *p = (u64 *) k.v;
p < (u64 *) bkey_val_end(k);
p++)
*p = swab64(*p);
}
static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p)
{
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, p);
switch (acc_k.type) {
case BCH_DISK_ACCOUNTING_replicas:
unsafe_memcpy(r, &acc_k.replicas,
replicas_entry_bytes(&acc_k.replicas),
"variable length struct");
return true;
default:
return false;
}
}
static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p)
{
struct bch_replicas_padded r;
return accounting_to_replicas(&r.e, p)
? bch2_mark_replicas(c, &r.e)
: 0;
}
/*
* Ensure accounting keys being updated are present in the superblock, when
* applicable (i.e. replicas updates)
*/
int bch2_accounting_update_sb(struct btree_trans *trans)
{
for (struct jset_entry *i = trans->journal_entries;
i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
i = vstruct_next(i))
if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) {
int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p);
if (ret)
return ret;
}
return 0;
}
static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a)
{
struct bch_accounting_mem *acc = &c->accounting;
/* raced with another insert, already present: */
if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
accounting_pos_cmp, &a.k->p) < acc->k.nr)
return 0;
struct accounting_mem_entry n = {
.pos = a.k->p,
.bversion = a.k->bversion,
.nr_counters = bch2_accounting_counters(a.k),
.v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
sizeof(u64), GFP_KERNEL),
};
if (!n.v[0])
goto err;
if (acc->gc_running) {
n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
sizeof(u64), GFP_KERNEL);
if (!n.v[1])
goto err;
}
if (darray_push(&acc->k, n))
goto err;
eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
accounting_pos_cmp, NULL);
return 0;
err:
free_percpu(n.v[1]);
free_percpu(n.v[0]);
return -BCH_ERR_ENOMEM_disk_accounting;
}
int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a,
enum bch_accounting_mode mode)
{
struct bch_replicas_padded r;
if (mode != BCH_ACCOUNTING_read &&
accounting_to_replicas(&r.e, a.k->p) &&
!bch2_replicas_marked_locked(c, &r.e))
return -BCH_ERR_btree_insert_need_mark_replicas;
percpu_up_read(&c->mark_lock);
percpu_down_write(&c->mark_lock);
int ret = __bch2_accounting_mem_insert(c, a);
percpu_up_write(&c->mark_lock);
percpu_down_read(&c->mark_lock);
return ret;
}
static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e)
{
for (unsigned i = 0; i < e->nr_counters; i++)
if (percpu_u64_get(e->v[0] + i) ||
(e->v[1] &&
percpu_u64_get(e->v[1] + i)))
return false;
return true;
}
void bch2_accounting_mem_gc(struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
percpu_down_write(&c->mark_lock);
struct accounting_mem_entry *dst = acc->k.data;
darray_for_each(acc->k, src) {
if (accounting_mem_entry_is_zero(src)) {
free_percpu(src->v[0]);
free_percpu(src->v[1]);
} else {
*dst++ = *src;
}
}
acc->k.nr = dst - acc->k.data;
eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
accounting_pos_cmp, NULL);
percpu_up_write(&c->mark_lock);
}
/*
* Read out accounting keys for replicas entries, as an array of
* bch_replicas_usage entries.
*
* Note: this may be deprecated/removed at smoe point in the future and replaced
* with something more general, it exists to support the ioctl used by the
* 'bcachefs fs usage' command.
*/
int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage)
{
struct bch_accounting_mem *acc = &c->accounting;
int ret = 0;
darray_init(usage);
percpu_down_read(&c->mark_lock);
darray_for_each(acc->k, i) {
struct {
struct bch_replicas_usage r;
u8 pad[BCH_BKEY_PTRS_MAX];
} u;
if (!accounting_to_replicas(&u.r.r, i->pos))
continue;
u64 sectors;
bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false);
u.r.sectors = sectors;
ret = darray_make_room(usage, replicas_usage_bytes(&u.r));
if (ret)
break;
memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r));
usage->nr += replicas_usage_bytes(&u.r);
}
percpu_up_read(&c->mark_lock);
if (ret)
darray_exit(usage);
return ret;
}
int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask)
{
struct bch_accounting_mem *acc = &c->accounting;
int ret = 0;
darray_init(out_buf);
percpu_down_read(&c->mark_lock);
darray_for_each(acc->k, i) {
struct disk_accounting_pos a_p;
bpos_to_disk_accounting_pos(&a_p, i->pos);
if (!(accounting_types_mask & BIT(a_p.type)))
continue;
ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) +
sizeof(u64) * i->nr_counters);
if (ret)
break;
struct bkey_i_accounting *a_out =
bkey_accounting_init((void *) &darray_top(*out_buf));
set_bkey_val_u64s(&a_out->k, i->nr_counters);
a_out->k.p = i->pos;
bch2_accounting_mem_read_counters(acc, i - acc->k.data,
a_out->v.d, i->nr_counters, false);
if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out)))
out_buf->nr += bkey_bytes(&a_out->k);
}
percpu_up_read(&c->mark_lock);
if (ret)
darray_exit(out_buf);
return ret;
}
void bch2_fs_accounting_to_text(struct printbuf *out, struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
percpu_down_read(&c->mark_lock);
out->atomic++;
eytzinger0_for_each(i, acc->k.nr) {
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, acc->k.data[i].pos);
bch2_accounting_key_to_text(out, &acc_k);
u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
prt_str(out, ":");
for (unsigned j = 0; j < acc->k.data[i].nr_counters; j++)
prt_printf(out, " %llu", v[j]);
prt_newline(out);
}
--out->atomic;
percpu_up_read(&c->mark_lock);
}
static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc)
{
darray_for_each(acc->k, e) {
free_percpu(e->v[gc]);
e->v[gc] = NULL;
}
}
int bch2_gc_accounting_start(struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
int ret = 0;
percpu_down_write(&c->mark_lock);
darray_for_each(acc->k, e) {
e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64),
sizeof(u64), GFP_KERNEL);
if (!e->v[1]) {
bch2_accounting_free_counters(acc, true);
ret = -BCH_ERR_ENOMEM_disk_accounting;
break;
}
}
acc->gc_running = !ret;
percpu_up_write(&c->mark_lock);
return ret;
}
int bch2_gc_accounting_done(struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
struct btree_trans *trans = bch2_trans_get(c);
struct printbuf buf = PRINTBUF;
struct bpos pos = POS_MIN;
int ret = 0;
percpu_down_write(&c->mark_lock);
while (1) {
unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
accounting_pos_cmp, &pos);
if (idx >= acc->k.nr)
break;
struct accounting_mem_entry *e = acc->k.data + idx;
pos = bpos_successor(e->pos);
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, e->pos);
if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
continue;
u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS];
u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS];
unsigned nr = e->nr_counters;
bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false);
bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true);
if (memcmp(dst_v, src_v, nr * sizeof(u64))) {
printbuf_reset(&buf);
prt_str(&buf, "accounting mismatch for ");
bch2_accounting_key_to_text(&buf, &acc_k);
prt_str(&buf, ": got");
for (unsigned j = 0; j < nr; j++)
prt_printf(&buf, " %llu", dst_v[j]);
prt_str(&buf, " should be");
for (unsigned j = 0; j < nr; j++)
prt_printf(&buf, " %llu", src_v[j]);
for (unsigned j = 0; j < nr; j++)
src_v[j] -= dst_v[j];
if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) {
percpu_up_write(&c->mark_lock);
ret = commit_do(trans, NULL, NULL, 0,
bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false));
percpu_down_write(&c->mark_lock);
if (ret)
goto err;
if (!test_bit(BCH_FS_may_go_rw, &c->flags)) {
memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
accounting_key_init(&k_i.k, &acc_k, src_v, nr);
bch2_accounting_mem_mod_locked(trans,
bkey_i_to_s_c_accounting(&k_i.k),
BCH_ACCOUNTING_normal);
preempt_disable();
struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);
struct bch_fs_usage_base *src = &trans->fs_usage_delta;
acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));
preempt_enable();
}
}
}
}
err:
fsck_err:
percpu_up_write(&c->mark_lock);
printbuf_exit(&buf);
bch2_trans_put(trans);
bch_err_fn(c, ret);
return ret;
}
static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k)
{
struct bch_fs *c = trans->c;
if (k.k->type != KEY_TYPE_accounting)
return 0;
percpu_down_read(&c->mark_lock);
int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k),
BCH_ACCOUNTING_read);
percpu_up_read(&c->mark_lock);
return ret;
}
/*
* At startup time, initialize the in memory accounting from the btree (and
* journal)
*/
int bch2_accounting_read(struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
struct btree_trans *trans = bch2_trans_get(c);
struct printbuf buf = PRINTBUF;
int ret = for_each_btree_key(trans, iter,
BTREE_ID_accounting, POS_MIN,
BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({
struct bkey u;
struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u);
accounting_read_key(trans, k);
}));
if (ret)
goto err;
struct journal_keys *keys = &c->journal_keys;
struct journal_key *dst = keys->data;
move_gap(keys, keys->nr);
darray_for_each(*keys, i) {
if (i->k->k.type == KEY_TYPE_accounting) {
struct bkey_s_c k = bkey_i_to_s_c(i->k);
unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr,
sizeof(acc->k.data[0]),
accounting_pos_cmp, &k.k->p);
bool applied = idx < acc->k.nr &&
bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0;
if (applied)
continue;
if (i + 1 < &darray_top(*keys) &&
i[1].k->k.type == KEY_TYPE_accounting &&
!journal_key_cmp(i, i + 1)) {
WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0);
i[1].journal_seq = i[0].journal_seq;
bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k),
bkey_s_c_to_accounting(k));
continue;
}
ret = accounting_read_key(trans, k);
if (ret)
goto err;
}
*dst++ = *i;
}
keys->gap = keys->nr = dst - keys->data;
percpu_down_read(&c->mark_lock);
for (unsigned i = 0; i < acc->k.nr; i++) {
u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
if (bch2_is_zero(v, sizeof(v[0]) * acc->k.data[i].nr_counters))
continue;
struct bch_replicas_padded r;
if (!accounting_to_replicas(&r.e, acc->k.data[i].pos))
continue;
/*
* If the replicas entry is invalid it'll get cleaned up by
* check_allocations:
*/
if (bch2_replicas_entry_validate(&r.e, c, &buf))
continue;
struct disk_accounting_pos k;
bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);
if (fsck_err_on(!bch2_replicas_marked_locked(c, &r.e),
trans, accounting_replicas_not_marked,
"accounting not marked in superblock replicas\n %s",
(printbuf_reset(&buf),
bch2_accounting_key_to_text(&buf, &k),
buf.buf))) {
/*
* We're not RW yet and still single threaded, dropping
* and retaking lock is ok:
*/
percpu_up_read(&c->mark_lock);
ret = bch2_mark_replicas(c, &r.e);
if (ret)
goto fsck_err;
percpu_down_read(&c->mark_lock);
}
}
preempt_disable();
struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage);
for (unsigned i = 0; i < acc->k.nr; i++) {
struct disk_accounting_pos k;
bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);
u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
switch (k.type) {
case BCH_DISK_ACCOUNTING_persistent_reserved:
usage->reserved += v[0] * k.persistent_reserved.nr_replicas;
break;
case BCH_DISK_ACCOUNTING_replicas:
fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]);
break;
case BCH_DISK_ACCOUNTING_dev_data_type:
rcu_read_lock();
struct bch_dev *ca = bch2_dev_rcu(c, k.dev_data_type.dev);
if (ca) {
struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type];
percpu_u64_set(&d->buckets, v[0]);
percpu_u64_set(&d->sectors, v[1]);
percpu_u64_set(&d->fragmented, v[2]);
if (k.dev_data_type.data_type == BCH_DATA_sb ||
k.dev_data_type.data_type == BCH_DATA_journal)
usage->hidden += v[0] * ca->mi.bucket_size;
}
rcu_read_unlock();
break;
}
}
preempt_enable();
fsck_err:
percpu_up_read(&c->mark_lock);
err:
printbuf_exit(&buf);
bch2_trans_put(trans);
bch_err_fn(c, ret);
return ret;
}
int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev)
{
return bch2_trans_run(c,
bch2_btree_write_buffer_flush_sync(trans) ?:
for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN,
BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({
struct disk_accounting_pos acc;
bpos_to_disk_accounting_pos(&acc, k.k->p);
acc.type == BCH_DISK_ACCOUNTING_dev_data_type &&
acc.dev_data_type.dev == dev
? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0)
: 0;
})) ?:
bch2_btree_write_buffer_flush_sync(trans));
}
int bch2_dev_usage_init(struct bch_dev *ca, bool gc)
{
struct bch_fs *c = ca->fs;
struct disk_accounting_pos acc = {
.type = BCH_DISK_ACCOUNTING_dev_data_type,
.dev_data_type.dev = ca->dev_idx,
.dev_data_type.data_type = BCH_DATA_free,
};
u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 };
int ret = bch2_trans_do(c, NULL, NULL, 0,
bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc));
bch_err_fn(c, ret);
return ret;
}
void bch2_verify_accounting_clean(struct bch_fs *c)
{
bool mismatch = false;
struct bch_fs_usage_base base = {}, base_inmem = {};
bch2_trans_run(c,
for_each_btree_key(trans, iter,
BTREE_ID_accounting, POS_MIN,
BTREE_ITER_all_snapshots, k, ({
u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k);
unsigned nr = bch2_accounting_counters(k.k);
struct disk_accounting_pos acc_k;
bpos_to_disk_accounting_pos(&acc_k, k.k->p);
if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
continue;
if (acc_k.type == BCH_DISK_ACCOUNTING_inum)
continue;
bch2_accounting_mem_read(c, k.k->p, v, nr);
if (memcmp(a.v->d, v, nr * sizeof(u64))) {
struct printbuf buf = PRINTBUF;
bch2_bkey_val_to_text(&buf, c, k);
prt_str(&buf, " !=");
for (unsigned j = 0; j < nr; j++)
prt_printf(&buf, " %llu", v[j]);
pr_err("%s", buf.buf);
printbuf_exit(&buf);
mismatch = true;
}
switch (acc_k.type) {
case BCH_DISK_ACCOUNTING_persistent_reserved:
base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];
break;
case BCH_DISK_ACCOUNTING_replicas:
fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]);
break;
case BCH_DISK_ACCOUNTING_dev_data_type: {
rcu_read_lock();
struct bch_dev *ca = bch2_dev_rcu(c, acc_k.dev_data_type.dev);
if (!ca) {
rcu_read_unlock();
continue;
}
v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets);
v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors);
v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented);
rcu_read_unlock();
if (memcmp(a.v->d, v, 3 * sizeof(u64))) {
struct printbuf buf = PRINTBUF;
bch2_bkey_val_to_text(&buf, c, k);
prt_str(&buf, " in mem");
for (unsigned j = 0; j < nr; j++)
prt_printf(&buf, " %llu", v[j]);
pr_err("dev accounting mismatch: %s", buf.buf);
printbuf_exit(&buf);
mismatch = true;
}
}
}
0;
})));
acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64));
#define check(x) \
if (base.x != base_inmem.x) { \
pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \
mismatch = true; \
}
//check(hidden);
check(btree);
check(data);
check(cached);
check(reserved);
check(nr_inodes);
WARN_ON(mismatch);
}
void bch2_accounting_gc_free(struct bch_fs *c)
{
lockdep_assert_held(&c->mark_lock);
struct bch_accounting_mem *acc = &c->accounting;
bch2_accounting_free_counters(acc, true);
acc->gc_running = false;
}
void bch2_fs_accounting_exit(struct bch_fs *c)
{
struct bch_accounting_mem *acc = &c->accounting;
bch2_accounting_free_counters(acc, false);
darray_exit(&acc->k);
}
Codebase Browser
linux