linux/fs/bcachefs/fs-io-direct.c

// SPDX-License-Identifier: GPL-2.0
#ifndef NO_BCACHEFS_FS

#include "bcachefs.h"
#include "alloc_foreground.h"
#include "fs.h"
#include "fs-io.h"
#include "fs-io-direct.h"
#include "fs-io-pagecache.h"
#include "io_read.h"
#include "io_write.h"

#include <linux/kthread.h>
#include <linux/pagemap.h>
#include <linux/prefetch.h>
#include <linux/task_io_accounting_ops.h>

/* O_DIRECT reads */

struct dio_read { … };

static void bio_check_or_release(struct bio *bio, bool check_dirty)
{ … }

static CLOSURE_CALLBACK(bch2_dio_read_complete)
{ … }

static void bch2_direct_IO_read_endio(struct bio *bio)
{ … }

static void bch2_direct_IO_read_split_endio(struct bio *bio)
{ … }

static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter)
{ … }

ssize_t bch2_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{ … }

/* O_DIRECT writes */

struct dio_write { … };

static bool bch2_check_range_allocated(struct bch_fs *c, subvol_inum inum,
				       u64 offset, u64 size,
				       unsigned nr_replicas, bool compressed)
{ … }

static noinline bool bch2_dio_write_check_allocated(struct dio_write *dio)
{ … }

static void bch2_dio_write_loop_async(struct bch_write_op *);
static __always_inline long bch2_dio_write_done(struct dio_write *dio);

/*
 * We're going to return -EIOCBQUEUED, but we haven't finished consuming the
 * iov_iter yet, so we need to stash a copy of the iovec: it might be on the
 * caller's stack, we're not guaranteed that it will live for the duration of
 * the IO:
 */
static noinline int bch2_dio_write_copy_iov(struct dio_write *dio)
{ … }

static CLOSURE_CALLBACK(bch2_dio_write_flush_done)
{ … }

static noinline void bch2_dio_write_flush(struct dio_write *dio)
{ … }

static __always_inline long bch2_dio_write_done(struct dio_write *dio)
{ … }

static __always_inline void bch2_dio_write_end(struct dio_write *dio)
{ … }

static __always_inline long bch2_dio_write_loop(struct dio_write *dio)
{ … }

static noinline __cold void bch2_dio_write_continue(struct dio_write *dio)
{ … }

static void bch2_dio_write_loop_async(struct bch_write_op *op)
{ … }

ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
{ … }

void bch2_fs_fs_io_direct_exit(struct bch_fs *c)
{ … }

int bch2_fs_fs_io_direct_init(struct bch_fs *c)
{ … }

#endif /* NO_BCACHEFS_FS */