// SPDX-License-Identifier: GPL-2.0-only /* * linux/fs/nfs/pagelist.c * * A set of helper functions for managing NFS read and write requests. * The main purpose of these routines is to provide support for the * coalescing of several requests into a single RPC call. * * Copyright 2000, 2001 (c) Trond Myklebust <[email protected]> * */ #include <linux/slab.h> #include <linux/file.h> #include <linux/sched.h> #include <linux/sunrpc/clnt.h> #include <linux/nfs.h> #include <linux/nfs3.h> #include <linux/nfs4.h> #include <linux/nfs_fs.h> #include <linux/nfs_page.h> #include <linux/nfs_mount.h> #include <linux/export.h> #include <linux/filelock.h> #include "internal.h" #include "pnfs.h" #include "nfstrace.h" #include "fscache.h" #define NFSDBG_FACILITY … static struct kmem_cache *nfs_page_cachep; static const struct rpc_call_ops nfs_pgio_common_ops; struct nfs_page_iter_page { … }; static void nfs_page_iter_page_init(struct nfs_page_iter_page *i, const struct nfs_page *req) { … } static void nfs_page_iter_page_advance(struct nfs_page_iter_page *i, size_t sz) { … } static struct page *nfs_page_iter_page_get(struct nfs_page_iter_page *i) { … } static struct nfs_pgio_mirror * nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx) { … } struct nfs_pgio_mirror * nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc) { … } EXPORT_SYMBOL_GPL(…); static u32 nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx) { … } void nfs_pgheader_init(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr, void (*release)(struct nfs_pgio_header *hdr)) { … } EXPORT_SYMBOL_GPL(…); void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos) { … } static inline struct nfs_page *nfs_page_alloc(void) { … } static inline void nfs_page_free(struct nfs_page *p) { … } /** * nfs_iocounter_wait - wait for i/o to complete * @l_ctx: nfs_lock_context with io_counter to use * * returns -ERESTARTSYS if interrupted by a fatal signal. * Otherwise returns 0 once the io_count hits 0. */ int nfs_iocounter_wait(struct nfs_lock_context *l_ctx) { … } /** * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O * to complete * @task: the rpc_task that should wait * @l_ctx: nfs_lock_context with io_counter to check * * Returns true if there is outstanding I/O to wait on and the * task has been put to sleep. */ bool nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx) { … } EXPORT_SYMBOL_GPL(…); /* * nfs_page_set_headlock - set the request PG_HEADLOCK * @req: request that is to be locked * * this lock must be held when modifying req->wb_head * * return 0 on success, < 0 on error */ int nfs_page_set_headlock(struct nfs_page *req) { … } /* * nfs_page_clear_headlock - clear the request PG_HEADLOCK * @req: request that is to be locked */ void nfs_page_clear_headlock(struct nfs_page *req) { … } /* * nfs_page_group_lock - lock the head of the page group * @req: request in group that is to be locked * * this lock must be held when traversing or modifying the page * group list * * return 0 on success, < 0 on error */ int nfs_page_group_lock(struct nfs_page *req) { … } /* * nfs_page_group_unlock - unlock the head of the page group * @req: request in group that is to be unlocked */ void nfs_page_group_unlock(struct nfs_page *req) { … } /* * nfs_page_group_sync_on_bit_locked * * must be called with page group lock held */ static bool nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit) { … } /* * nfs_page_group_sync_on_bit - set bit on current request, but only * return true if the bit is set for all requests in page group * @req - request in page group * @bit - PG_* bit that is used to sync page group */ bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit) { … } /* * nfs_page_group_init - Initialize the page group linkage for @req * @req - a new nfs request * @prev - the previous request in page group, or NULL if @req is the first * or only request in the group (the head). */ static inline void nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev) { … } /* * nfs_page_group_destroy - sync the destruction of page groups * @req - request that no longer needs the page group * * releases the page group reference from each member once all * members have called this function. */ static void nfs_page_group_destroy(struct kref *kref) { … } static struct nfs_page *nfs_page_create(struct nfs_lock_context *l_ctx, unsigned int pgbase, pgoff_t index, unsigned int offset, unsigned int count) { … } static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio) { … } static void nfs_page_assign_page(struct nfs_page *req, struct page *page) { … } /** * nfs_page_create_from_page - Create an NFS read/write request. * @ctx: open context to use * @page: page to write * @pgbase: starting offset within the page for the write * @offset: file offset for the write * @count: number of bytes to read/write * * The page must be locked by the caller. This makes sure we never * create two different requests for the same page. * User should ensure it is safe to sleep in this function. */ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, struct page *page, unsigned int pgbase, loff_t offset, unsigned int count) { … } /** * nfs_page_create_from_folio - Create an NFS read/write request. * @ctx: open context to use * @folio: folio to write * @offset: starting offset within the folio for the write * @count: number of bytes to read/write * * The page must be locked by the caller. This makes sure we never * create two different requests for the same page. * User should ensure it is safe to sleep in this function. */ struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, struct folio *folio, unsigned int offset, unsigned int count) { … } static struct nfs_page * nfs_create_subreq(struct nfs_page *req, unsigned int pgbase, unsigned int offset, unsigned int count) { … } /** * nfs_unlock_request - Unlock request and wake up sleepers. * @req: pointer to request */ void nfs_unlock_request(struct nfs_page *req) { … } /** * nfs_unlock_and_release_request - Unlock request and release the nfs_page * @req: pointer to request */ void nfs_unlock_and_release_request(struct nfs_page *req) { … } /* * nfs_clear_request - Free up all resources allocated to the request * @req: * * Release page and open context resources associated with a read/write * request after it has completed. */ static void nfs_clear_request(struct nfs_page *req) { … } /** * nfs_free_request - Release the count on an NFS read/write request * @req: request to release * * Note: Should never be called with the spinlock held! */ void nfs_free_request(struct nfs_page *req) { … } void nfs_release_request(struct nfs_page *req) { … } EXPORT_SYMBOL_GPL(…); /* * nfs_generic_pg_test - determine if requests can be coalesced * @desc: pointer to descriptor * @prev: previous request in desc, or NULL * @req: this request * * Returns zero if @req cannot be coalesced into @desc, otherwise it returns * the size of the request. */ size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, struct nfs_page *prev, struct nfs_page *req) { … } EXPORT_SYMBOL_GPL(…); struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops) { … } EXPORT_SYMBOL_GPL(…); /** * nfs_pgio_data_destroy - make @hdr suitable for reuse * * Frees memory and releases refs from nfs_generic_pgio, so that it may * be called again. * * @hdr: A header that has had nfs_generic_pgio called */ static void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr) { … } /* * nfs_pgio_header_free - Free a read or write header * @hdr: The header to free */ void nfs_pgio_header_free(struct nfs_pgio_header *hdr) { … } EXPORT_SYMBOL_GPL(…); /** * nfs_pgio_rpcsetup - Set up arguments for a pageio call * @hdr: The pageio hdr * @pgbase: base * @count: Number of bytes to read * @how: How to commit data (writes only) * @cinfo: Commit information for the call (writes only) */ static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr, unsigned int pgbase, unsigned int count, int how, struct nfs_commit_info *cinfo) { … } /** * nfs_pgio_prepare - Prepare pageio hdr to go over the wire * @task: The current task * @calldata: pageio header to prepare */ static void nfs_pgio_prepare(struct rpc_task *task, void *calldata) { … } int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, const struct cred *cred, const struct nfs_rpc_ops *rpc_ops, const struct rpc_call_ops *call_ops, int how, int flags) { … } EXPORT_SYMBOL_GPL(…); /** * nfs_pgio_error - Clean up from a pageio error * @hdr: pageio header */ static void nfs_pgio_error(struct nfs_pgio_header *hdr) { … } /** * nfs_pgio_release - Release pageio data * @calldata: The pageio header to release */ static void nfs_pgio_release(void *calldata) { … } static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror, unsigned int bsize) { … } /** * nfs_pageio_init - initialise a page io descriptor * @desc: pointer to descriptor * @inode: pointer to inode * @pg_ops: pointer to pageio operations * @compl_ops: pointer to pageio completion operations * @rw_ops: pointer to nfs read/write operations * @bsize: io block size * @io_flags: extra parameters for the io function */ void nfs_pageio_init(struct nfs_pageio_descriptor *desc, struct inode *inode, const struct nfs_pageio_ops *pg_ops, const struct nfs_pgio_completion_ops *compl_ops, const struct nfs_rw_ops *rw_ops, size_t bsize, int io_flags) { … } /** * nfs_pgio_result - Basic pageio error handling * @task: The task that ran * @calldata: Pageio header to check */ static void nfs_pgio_result(struct rpc_task *task, void *calldata) { … } /* * Create an RPC task for the given read or write request and kick it. * The page must have been locked by the caller. * * It may happen that the page we're passed is not marked dirty. * This is the case if nfs_updatepage detects a conflicting request * that has been written but not committed. */ int nfs_generic_pgio(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) { … } EXPORT_SYMBOL_GPL(…); static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) { … } static struct nfs_pgio_mirror * nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc, unsigned int mirror_count) { … } /* * nfs_pageio_setup_mirroring - determine if mirroring is to be used * by calling the pg_get_mirror_count op */ static void nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) { … } static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio) { … } static bool nfs_match_lock_context(const struct nfs_lock_context *l1, const struct nfs_lock_context *l2) { … } static bool nfs_page_is_contiguous(const struct nfs_page *prev, const struct nfs_page *req) { … } /** * nfs_coalesce_size - test two requests for compatibility * @prev: pointer to nfs_page * @req: pointer to nfs_page * @pgio: pointer to nfs_pagio_descriptor * * The nfs_page structures 'prev' and 'req' are compared to ensure that the * page data area they describe is contiguous, and that their RPC * credentials, NFSv4 open state, and lockowners are the same. * * Returns size of the request that can be coalesced */ static unsigned int nfs_coalesce_size(struct nfs_page *prev, struct nfs_page *req, struct nfs_pageio_descriptor *pgio) { … } /** * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list. * @desc: destination io descriptor * @req: request * * If the request 'req' was successfully coalesced into the existing list * of pages 'desc', it returns the size of req. */ static unsigned int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { … } /* * Helper for nfs_pageio_add_request and nfs_pageio_complete */ static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc) { … } static void nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { … } /** * __nfs_pageio_add_request - Attempt to coalesce a request into a page list. * @desc: destination io descriptor * @req: request * * This may split a request into subrequests which are all part of the * same page group. If so, it will submit @req as the last one, to ensure * the pointer to @req is still valid in case of failure. * * Returns true if the request 'req' was successfully coalesced into the * existing list of pages 'desc'. */ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { … } static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc) { … } static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { … } static void nfs_pageio_error_cleanup(struct nfs_pageio_descriptor *desc) { … } int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, struct nfs_page *req) { … } /* * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an * nfs_pageio_descriptor * @desc: pointer to io descriptor * @mirror_idx: pointer to mirror index */ static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc, u32 mirror_idx) { … } /* * nfs_pageio_resend - Transfer requests to new descriptor and resend * @hdr - the pgio header to move request from * @desc - the pageio descriptor to add requests to * * Try to move each request (nfs_page) from @hdr to @desc then attempt * to send them. * * Returns 0 on success and < 0 on error. */ int nfs_pageio_resend(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) { … } EXPORT_SYMBOL_GPL(…); /** * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor * @desc: pointer to io descriptor */ void nfs_pageio_complete(struct nfs_pageio_descriptor *desc) { … } /** * nfs_pageio_cond_complete - Conditional I/O completion * @desc: pointer to io descriptor * @index: page index * * It is important to ensure that processes don't try to take locks * on non-contiguous ranges of pages as that might deadlock. This * function should be called before attempting to wait on a locked * nfs_page. It will complete the I/O if the page index 'index' * is not contiguous with the existing list of pages in 'desc'. */ void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index) { … } /* * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1) */ void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio) { … } int __init nfs_init_nfspagecache(void) { … } void nfs_destroy_nfspagecache(void) { … } static const struct rpc_call_ops nfs_pgio_common_ops = …; const struct nfs_pageio_ops nfs_pgio_rw_ops = …;
Codebase Browser
linux