// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2016-2018 Oracle. All rights reserved. * * Use the core R/W API to move RPC-over-RDMA Read and Write chunks. */ #include <rdma/rw.h> #include <linux/sunrpc/xdr.h> #include <linux/sunrpc/rpc_rdma.h> #include <linux/sunrpc/svc_rdma.h> #include "xprt_rdma.h" #include <trace/events/rpcrdma.h> static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); /* Each R/W context contains state for one chain of RDMA Read or * Write Work Requests. * * Each WR chain handles a single contiguous server-side buffer, * because scatterlist entries after the first have to start on * page alignment. xdr_buf iovecs cannot guarantee alignment. * * Each WR chain handles only one R_key. Each RPC-over-RDMA segment * from a client may contain a unique R_key, so each WR chain moves * up to one segment at a time. * * The scatterlist makes this data structure over 4KB in size. To * make it less likely to fail, and to handle the allocation for * smaller I/O requests without disabling bottom-halves, these * contexts are created on demand, but cached and reused until the * controlling svcxprt_rdma is destroyed. */ struct svc_rdma_rw_ctxt { … }; static inline struct svc_rdma_rw_ctxt * svc_rdma_next_ctxt(struct list_head *list) { … } static struct svc_rdma_rw_ctxt * svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) { … } static void __svc_rdma_put_rw_ctxt(struct svc_rdma_rw_ctxt *ctxt, struct llist_head *list) { … } static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, struct svc_rdma_rw_ctxt *ctxt) { … } /** * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts * @rdma: transport about to be destroyed * */ void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) { … } /** * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O * @rdma: controlling transport instance * @ctxt: R/W context to prepare * @offset: RDMA offset * @handle: RDMA tag/handle * @direction: I/O direction * * Returns on success, the number of WQEs that will be needed * on the workqueue, or a negative errno. */ static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, struct svc_rdma_rw_ctxt *ctxt, u64 offset, u32 handle, enum dma_data_direction direction) { … } /** * svc_rdma_cc_init - Initialize an svc_rdma_chunk_ctxt * @rdma: controlling transport instance * @cc: svc_rdma_chunk_ctxt to be initialized */ void svc_rdma_cc_init(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc) { … } /** * svc_rdma_cc_release - Release resources held by a svc_rdma_chunk_ctxt * @rdma: controlling transport instance * @cc: svc_rdma_chunk_ctxt to be released * @dir: DMA direction */ void svc_rdma_cc_release(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc, enum dma_data_direction dir) { … } static struct svc_rdma_write_info * svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, const struct svc_rdma_chunk *chunk) { … } static void svc_rdma_write_info_free_async(struct work_struct *work) { … } static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) { … } /** * svc_rdma_reply_chunk_release - Release Reply chunk I/O resources * @rdma: controlling transport * @ctxt: Send context that is being released */ void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt) { … } /** * svc_rdma_reply_done - Reply chunk Write completion handler * @cq: controlling Completion Queue * @wc: Work Completion report * * Pages under I/O are released by a subsequent Send completion. */ static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc) { … } /** * svc_rdma_write_done - Write chunk completion * @cq: controlling Completion Queue * @wc: Work Completion * * Pages under I/O are freed by a subsequent Send completion. */ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) { … } /** * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx * @cq: controlling Completion Queue * @wc: Work Completion * */ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) { … } /* * Assumptions: * - If ib_post_send() succeeds, only one completion is expected, * even if one or more WRs are flushed. This is true when posting * an rdma_rw_ctx or when posting a single signaled WR. */ static int svc_rdma_post_chunk_ctxt(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc) { … } /* Build and DMA-map an SGL that covers one kvec in an xdr_buf */ static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, unsigned int len, struct svc_rdma_rw_ctxt *ctxt) { … } /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. */ static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, unsigned int remaining, struct svc_rdma_rw_ctxt *ctxt) { … } /* Construct RDMA Write WRs to send a portion of an xdr_buf containing * an RPC Reply. */ static int svc_rdma_build_writes(struct svc_rdma_write_info *info, void (*constructor)(struct svc_rdma_write_info *info, unsigned int len, struct svc_rdma_rw_ctxt *ctxt), unsigned int remaining) { … } /** * svc_rdma_iov_write - Construct RDMA Writes from an iov * @info: pointer to write arguments * @iov: kvec to write * * Returns: * On success, returns zero * %-E2BIG if the client-provided Write chunk is too small * %-ENOMEM if a resource has been exhausted * %-EIO if an rdma-rw error occurred */ static int svc_rdma_iov_write(struct svc_rdma_write_info *info, const struct kvec *iov) { … } /** * svc_rdma_pages_write - Construct RDMA Writes from pages * @info: pointer to write arguments * @xdr: xdr_buf with pages to write * @offset: offset into the content of @xdr * @length: number of bytes to write * * Returns: * On success, returns zero * %-E2BIG if the client-provided Write chunk is too small * %-ENOMEM if a resource has been exhausted * %-EIO if an rdma-rw error occurred */ static int svc_rdma_pages_write(struct svc_rdma_write_info *info, const struct xdr_buf *xdr, unsigned int offset, unsigned long length) { … } /** * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf * @xdr: xdr_buf to write * @data: pointer to write arguments * * Returns: * On success, returns zero * %-E2BIG if the client-provided Write chunk is too small * %-ENOMEM if a resource has been exhausted * %-EIO if an rdma-rw error occurred */ static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) { … } static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_chunk *chunk, const struct xdr_buf *xdr) { … } /** * svc_rdma_send_write_list - Send all chunks on the Write list * @rdma: controlling RDMA transport * @rctxt: Write list provisioned by the client * @xdr: xdr_buf containing an RPC Reply message * * Returns zero on success, or a negative errno if one or more * Write chunks could not be sent. */ int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, const struct svc_rdma_recv_ctxt *rctxt, const struct xdr_buf *xdr) { … } /** * svc_rdma_prepare_reply_chunk - Construct WR chain for writing the Reply chunk * @rdma: controlling RDMA transport * @write_pcl: Write chunk list provided by client * @reply_pcl: Reply chunk provided by client * @sctxt: Send WR resources * @xdr: xdr_buf containing an RPC Reply * * Returns a non-negative number of bytes the chunk consumed, or * %-E2BIG if the payload was larger than the Reply chunk, * %-EINVAL if client provided too many segments, * %-ENOMEM if rdma_rw context pool was exhausted, * %-ENOTCONN if posting failed (connection is lost), * %-EIO if rdma_rw initialization failed (DMA mapping, etc). */ int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_pcl *write_pcl, const struct svc_rdma_pcl *reply_pcl, struct svc_rdma_send_ctxt *sctxt, const struct xdr_buf *xdr) { … } /** * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment * @rqstp: RPC transaction context * @head: context for ongoing I/O * @segment: co-ordinates of remote memory to be read * * Returns: * %0: the Read WR chain was constructed successfully * %-EINVAL: there were not enough rq_pages to finish * %-ENOMEM: allocating a local resources failed * %-EIO: a DMA mapping error occurred */ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head, const struct svc_rdma_segment *segment) { … } /** * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk * @rqstp: RPC transaction context * @head: context for ongoing I/O * @chunk: Read chunk to pull * * Return values: * %0: the Read WR chain was constructed successfully * %-EINVAL: there were not enough resources to finish * %-ENOMEM: allocating a local resources failed * %-EIO: a DMA mapping error occurred */ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head, const struct svc_rdma_chunk *chunk) { … } /** * svc_rdma_copy_inline_range - Copy part of the inline content into pages * @rqstp: RPC transaction context * @head: context for ongoing I/O * @offset: offset into the Receive buffer of region to copy * @remaining: length of region to copy * * Take a page at a time from rqstp->rq_pages and copy the inline * content from the Receive buffer into that page. Update * head->rc_curpage and head->rc_pageoff so that the next RDMA Read * result will land contiguously with the copied content. * * Return values: * %0: Inline content was successfully copied * %-EINVAL: offset or length was incorrect */ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head, unsigned int offset, unsigned int remaining) { … } /** * svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks * @rqstp: RPC transaction context * @head: context for ongoing I/O * * The chunk data lands in rqstp->rq_arg as a series of contiguous pages, * like an incoming TCP call. * * Return values: * %0: RDMA Read WQEs were successfully built * %-EINVAL: client provided too many chunks or segments, * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ static noinline int svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … } /** * svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks * @rqstp: RPC transaction context * @head: context for ongoing I/O * * The chunk data lands in the page list of rqstp->rq_arg.pages. * * Currently NFSD does not look at the rqstp->rq_arg.tail[0] kvec. * Therefore, XDR round-up of the Read chunk and trailing * inline content must both be added at the end of the pagelist. * * Return values: * %0: RDMA Read WQEs were successfully built * %-EINVAL: client provided too many chunks or segments, * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ static int svc_rdma_read_data_item(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … } /** * svc_rdma_read_chunk_range - Build RDMA Read WRs for portion of a chunk * @rqstp: RPC transaction context * @head: context for ongoing I/O * @chunk: parsed Call chunk to pull * @offset: offset of region to pull * @length: length of region to pull * * Return values: * %0: RDMA Read WQEs were successfully built * %-EINVAL: there were not enough resources to finish * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head, const struct svc_rdma_chunk *chunk, unsigned int offset, unsigned int length) { … } /** * svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message * @rqstp: RPC transaction context * @head: context for ongoing I/O * * Return values: * %0: RDMA Read WQEs were successfully built * %-EINVAL: there were not enough resources to finish * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … } /** * svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message * @rqstp: RPC transaction context * @head: context for ongoing I/O * * The start of the data lands in the first page just after the * Transport header, and the rest lands in rqstp->rq_arg.pages. * * Assumptions: * - A PZRC is never sent in an RDMA_MSG message, though it's * allowed by spec. * * Return values: * %0: RDMA Read WQEs were successfully built * %-EINVAL: client provided too many chunks or segments, * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ static noinline int svc_rdma_read_special(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … } /* Pages under I/O have been copied to head->rc_pages. Ensure that * svc_xprt_release() does not put them when svc_rdma_recvfrom() * returns. This has to be done after all Read WRs are constructed * to properly handle a page that happens to be part of I/O on behalf * of two different RDMA segments. * * Note: if the subsequent post_send fails, these pages have already * been moved to head->rc_pages and thus will be cleaned up by * svc_rdma_recv_ctxt_put(). */ static void svc_rdma_clear_rqst_pages(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … } /** * svc_rdma_process_read_list - Pull list of Read chunks from the client * @rdma: controlling RDMA transport * @rqstp: set of pages to use as Read sink buffers * @head: pages under I/O collect here * * The RPC/RDMA protocol assumes that the upper layer's XDR decoders * pull each Read chunk as they decode an incoming RPC message. * * On Linux, however, the server needs to have a fully-constructed RPC * message in rqstp->rq_arg when there is a positive return code from * ->xpo_recvfrom. So the Read list is safety-checked immediately when * it is received, then here the whole Read list is pulled all at once. * The ingress RPC message is fully reconstructed once all associated * RDMA Reads have completed. * * Return values: * %1: all needed RDMA Reads were posted successfully, * %-EINVAL: client provided too many chunks or segments, * %-ENOMEM: rdma_rw context pool was exhausted, * %-ENOTCONN: posting failed (connection is lost), * %-EIO: rdma_rw initialization failed (DMA mapping, etc). */ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { … }
Codebase Browser
linux