// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2015, 2017 Oracle. All rights reserved. * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. */ /* Lightweight memory registration using Fast Registration Work * Requests (FRWR). * * FRWR features ordered asynchronous registration and invalidation * of arbitrarily-sized memory regions. This is the fastest and safest * but most complex memory registration mode. */ /* Normal operation * * A Memory Region is prepared for RDMA Read or Write using a FAST_REG * Work Request (frwr_map). When the RDMA operation is finished, this * Memory Region is invalidated using a LOCAL_INV Work Request * (frwr_unmap_async and frwr_unmap_sync). * * Typically FAST_REG Work Requests are not signaled, and neither are * RDMA Send Work Requests (with the exception of signaling occasionally * to prevent provider work queue overflows). This greatly reduces HCA * interrupt workload. */ /* Transport recovery * * frwr_map and frwr_unmap_* cannot run at the same time the transport * connect worker is running. The connect worker holds the transport * send lock, just as ->send_request does. This prevents frwr_map and * the connect worker from running concurrently. When a connection is * closed, the Receive completion queue is drained before the allowing * the connect worker to get control. This prevents frwr_unmap and the * connect worker from running concurrently. * * When the underlying transport disconnects, MRs that are in flight * are flushed and are likely unusable. Thus all MRs are destroyed. * New MRs are created on demand. */ #include <linux/sunrpc/svc_rdma.h> #include "xprt_rdma.h" #include <trace/events/rpcrdma.h> static void frwr_cid_init(struct rpcrdma_ep *ep, struct rpcrdma_mr *mr) { … } static void frwr_mr_unmap(struct rpcrdma_mr *mr) { … } /** * frwr_mr_release - Destroy one MR * @mr: MR allocated by frwr_mr_init * */ void frwr_mr_release(struct rpcrdma_mr *mr) { … } static void frwr_mr_put(struct rpcrdma_mr *mr) { … } /** * frwr_reset - Place MRs back on @req's free list * @req: request to reset * * Used after a failed marshal. For FRWR, this means the MRs * don't have to be fully released and recreated. * * NB: This is safe only as long as none of @req's MRs are * involved with an ongoing asynchronous FAST_REG or LOCAL_INV * Work Request. */ void frwr_reset(struct rpcrdma_req *req) { … } /** * frwr_mr_init - Initialize one MR * @r_xprt: controlling transport instance * @mr: generic MR to prepare for FRWR * * Returns zero if successful. Otherwise a negative errno * is returned. */ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) { … } /** * frwr_query_device - Prepare a transport for use with FRWR * @ep: endpoint to fill in * @device: RDMA device to query * * On success, sets: * ep->re_attr * ep->re_max_requests * ep->re_max_rdma_segs * ep->re_max_fr_depth * ep->re_mrtype * * Return values: * On success, returns zero. * %-EINVAL - the device does not support FRWR memory registration * %-ENOMEM - the device is not sufficiently capable for NFS/RDMA */ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) { … } /** * frwr_map - Register a memory region * @r_xprt: controlling transport * @seg: memory region co-ordinates * @nsegs: number of segments remaining * @writing: true when RDMA Write will be used * @xid: XID of RPC using the registered memory * @mr: MR to fill in * * Prepare a REG_MR Work Request to register a memory region * for remote access via RDMA READ or RDMA WRITE. * * Returns the next segment or a negative errno pointer. * On success, @mr is filled in. */ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, int nsegs, bool writing, __be32 xid, struct rpcrdma_mr *mr) { … } /** * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC * @cq: completion queue * @wc: WCE for a completed FastReg WR * * Each flushed MR gets destroyed after the QP has drained. */ static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc) { … } /** * frwr_send - post Send WRs containing the RPC Call message * @r_xprt: controlling transport instance * @req: prepared RPC Call * * For FRWR, chain any FastReg WRs to the Send WR. Only a * single ib_post_send call is needed to register memory * and then post the Send WR. * * Returns the return code from ib_post_send. * * Caller must hold the transport send lock to ensure that the * pointers to the transport's rdma_cm_id and QP are stable. */ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { … } /** * frwr_reminv - handle a remotely invalidated mr on the @mrs list * @rep: Received reply * @mrs: list of MRs to check * */ void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs) { … } static void frwr_mr_done(struct ib_wc *wc, struct rpcrdma_mr *mr) { … } /** * frwr_wc_localinv - Invoked by RDMA provider for a LOCAL_INV WC * @cq: completion queue * @wc: WCE for a completed LocalInv WR * */ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) { … } /** * frwr_wc_localinv_wake - Invoked by RDMA provider for a LOCAL_INV WC * @cq: completion queue * @wc: WCE for a completed LocalInv WR * * Awaken anyone waiting for an MR to finish being fenced. */ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) { … } /** * frwr_unmap_sync - invalidate memory regions that were registered for @req * @r_xprt: controlling transport instance * @req: rpcrdma_req with a non-empty list of MRs to process * * Sleeps until it is safe for the host CPU to access the previously mapped * memory regions. This guarantees that registered MRs are properly fenced * from the server before the RPC consumer accesses the data in them. It * also ensures proper Send flow control: waking the next RPC waits until * this RPC has relinquished all its Send Queue entries. */ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { … } /** * frwr_wc_localinv_done - Invoked by RDMA provider for a signaled LOCAL_INV WC * @cq: completion queue * @wc: WCE for a completed LocalInv WR * */ static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc) { … } /** * frwr_unmap_async - invalidate memory regions that were registered for @req * @r_xprt: controlling transport instance * @req: rpcrdma_req with a non-empty list of MRs to process * * This guarantees that registered MRs are properly fenced from the * server before the RPC consumer accesses the data in them. It also * ensures proper Send flow control: waking the next RPC waits until * this RPC has relinquished all its Send Queue entries. */ void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { … } /** * frwr_wp_create - Create an MR for padding Write chunks * @r_xprt: transport resources to use * * Return 0 on success, negative errno on failure. */ int frwr_wp_create(struct rpcrdma_xprt *r_xprt) { … }
Codebase Browser
linux