// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (c) 2014-2017 Oracle. All rights reserved. * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the BSD-type * license below: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials provided * with the distribution. * * Neither the name of the Network Appliance, Inc. nor the names of * its contributors may be used to endorse or promote products * derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * verbs.c * * Encapsulates the major functions managing: * o adapters * o endpoints * o connections * o buffer memory */ #include <linux/bitops.h> #include <linux/interrupt.h> #include <linux/slab.h> #include <linux/sunrpc/addr.h> #include <linux/sunrpc/svc_rdma.h> #include <linux/log2.h> #include <asm/barrier.h> #include <rdma/ib_cm.h> #include "xprt_rdma.h" #include <trace/events/rpcrdma.h> static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt); static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_sendctx *sc); static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt); static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt); static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); static void rpcrdma_ep_get(struct rpcrdma_ep *ep); static int rpcrdma_ep_put(struct rpcrdma_ep *ep); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction, int node); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); /* Wait for outstanding transport work to finish. ib_drain_qp * handles the drains in the wrong order for us, so open code * them here. */ static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) { … } /* Ensure xprt_force_disconnect() is invoked exactly once when a * connection is closed or lost. (The important thing is it needs * to be invoked "at least" once). */ void rpcrdma_force_disconnect(struct rpcrdma_ep *ep) { … } /** * rpcrdma_flush_disconnect - Disconnect on flushed completion * @r_xprt: transport to disconnect * @wc: work completion entry * * Must be called in process context. */ void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc) { … } /** * rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC * @cq: completion queue * @wc: WCE for a completed Send WR * */ static void rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) { … } /** * rpcrdma_wc_receive - Invoked by RDMA provider for each polled Receive WC * @cq: completion queue * @wc: WCE for a completed Receive WR * */ static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) { … } static void rpcrdma_update_cm_private(struct rpcrdma_ep *ep, struct rdma_conn_param *param) { … } /** * rpcrdma_cm_event_handler - Handle RDMA CM events * @id: rdma_cm_id on which an event has occurred * @event: details of the event * * Called with @id's mutex held. Returns 1 if caller should * destroy @id, otherwise 0. */ static int rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { … } static void rpcrdma_ep_removal_done(struct rpcrdma_notification *rn) { … } static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep) { … } static void rpcrdma_ep_destroy(struct kref *kref) { … } static noinline void rpcrdma_ep_get(struct rpcrdma_ep *ep) { … } /* Returns: * %0 if @ep still has a positive kref count, or * %1 if @ep was destroyed successfully. */ static noinline int rpcrdma_ep_put(struct rpcrdma_ep *ep) { … } static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_xprt_connect - Connect an unconnected transport * @r_xprt: controlling transport instance * * Returns 0 on success or a negative errno. */ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_xprt_disconnect - Disconnect underlying transport * @r_xprt: controlling transport instance * * Caller serializes. Either the transport send lock is held, * or we're being called to destroy the transport. * * On return, @r_xprt is completely divested of all hardware * resources and prepared for the next ->connect operation. */ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) { … } /* Fixed-size circular FIFO queue. This implementation is wait-free and * lock-free. * * Consumer is the code path that posts Sends. This path dequeues a * sendctx for use by a Send operation. Multiple consumer threads * are serialized by the RPC transport lock, which allows only one * ->send_request call at a time. * * Producer is the code path that handles Send completions. This path * enqueues a sendctx that has been completed. Multiple producer * threads are serialized by the ib_poll_cq() function. */ /* rpcrdma_sendctxs_destroy() assumes caller has already quiesced * queue activity, and rpcrdma_xprt_drain has flushed all remaining * Send requests. */ static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt) { … } static struct rpcrdma_sendctx *rpcrdma_sendctx_create(struct rpcrdma_ep *ep) { … } static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) { … } /* The sendctx queue is not guaranteed to have a size that is a * power of two, thus the helpers in circ_buf.h cannot be used. * The other option is to use modulus (%), which can be expensive. */ static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, unsigned long item) { … } /** * rpcrdma_sendctx_get_locked - Acquire a send context * @r_xprt: controlling transport instance * * Returns pointer to a free send completion context; or NULL if * the queue is empty. * * Usage: Called to acquire an SGE array before preparing a Send WR. * * The caller serializes calls to this function (per transport), and * provides an effective memory barrier that flushes the new value * of rb_sc_head. */ struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_sendctx_put_locked - Release a send context * @r_xprt: controlling transport instance * @sc: send context to release * * Usage: Called from Send completion to return a sendctxt * to the queue. * * The caller serializes calls to this function (per transport). */ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_sendctx *sc) { … } static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) { … } static void rpcrdma_mr_refresh_worker(struct work_struct *work) { … } /** * rpcrdma_mrs_refresh - Wake the MR refresh worker * @r_xprt: controlling transport instance * */ void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_req_create - Allocate an rpcrdma_req object * @r_xprt: controlling r_xprt * @size: initial size, in bytes, of send and receive buffers * * Returns an allocated and fully initialized rpcrdma_req or NULL. */ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size) { … } /** * rpcrdma_req_setup - Per-connection instance setup of an rpcrdma_req object * @r_xprt: controlling transport instance * @req: rpcrdma_req object to set up * * Returns zero on success, and a negative errno on failure. */ int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { … } /* ASSUMPTION: the rb_allreqs list is stable for the duration, * and thus can be walked without holding rb_lock. Eg. the * caller is holding the transport send lock to exclude * device removal or disconnection. */ static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt) { … } static void rpcrdma_req_reset(struct rpcrdma_req *req) { … } /* ASSUMPTION: the rb_allreqs list is stable for the duration, * and thus can be walked without holding rb_lock. Eg. the * caller is holding the transport send lock to exclude * device removal or disconnection. */ static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt) { … } static noinline struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt) { … } static void rpcrdma_rep_free(struct rpcrdma_rep *rep) { … } static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf) { … } /** * rpcrdma_rep_put - Release rpcrdma_rep back to free list * @buf: buffer pool * @rep: rep to release * */ void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep) { … } /* Caller must ensure the QP is quiescent (RQ is drained) before * invoking this function, to guarantee rb_all_reps is not * changing. */ static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt) { … } static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) { … } /** * rpcrdma_buffer_create - Create initial set of req/rep objects * @r_xprt: transport instance to (re)initialize * * Returns zero on success, otherwise a negative errno. */ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_req_destroy - Destroy an rpcrdma_req object * @req: unused object to be destroyed * * Relies on caller holding the transport send lock to protect * removing req->rl_all from buf->rb_all_reqs safely. */ void rpcrdma_req_destroy(struct rpcrdma_req *req) { … } /** * rpcrdma_mrs_destroy - Release all of a transport's MRs * @r_xprt: controlling transport instance * * Relies on caller holding the transport send lock to protect * removing mr->mr_list from req->rl_free_mrs safely. */ static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_buffer_destroy - Release all hw resources * @buf: root control block for resources * * ORDERING: relies on a prior rpcrdma_xprt_drain : * - No more Send or Receive completions can occur * - All MRs, reps, and reqs are returned to their free lists */ void rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) { … } /** * rpcrdma_mr_get - Allocate an rpcrdma_mr object * @r_xprt: controlling transport * * Returns an initialized rpcrdma_mr or NULL if no free * rpcrdma_mr objects are available. */ struct rpcrdma_mr * rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) { … } /** * rpcrdma_reply_put - Put reply buffers back into pool * @buffers: buffer pool * @req: object to return * */ void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) { … } /** * rpcrdma_buffer_get - Get a request buffer * @buffers: Buffer pool from which to obtain a buffer * * Returns a fresh rpcrdma_req, or NULL if none are available. */ struct rpcrdma_req * rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) { … } /** * rpcrdma_buffer_put - Put request/reply buffers back into pool * @buffers: buffer pool * @req: object to return * */ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) { … } /* Returns a pointer to a rpcrdma_regbuf object, or NULL. * * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for * receiving the payload of RDMA RECV operations. During Long Calls * or Replies they may be registered externally via frwr_map. */ static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction, int node) { … } static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) { … } /** * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer * @rb: regbuf to reallocate * @size: size of buffer to be allocated, in bytes * @flags: GFP flags * * Returns true if reallocation was successful. If false is * returned, @rb is left untouched. */ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) { … } /** * __rpcrdma_regbuf_dma_map - DMA-map a regbuf * @r_xprt: controlling transport instance * @rb: regbuf to be mapped * * Returns true if the buffer is now DMA mapped to @r_xprt's device */ bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_regbuf *rb) { … } static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb) { … } static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb) { … } /** * rpcrdma_post_recvs - Refill the Receive Queue * @r_xprt: controlling transport instance * @needed: current credit grant * */ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) { … }
Codebase Browser
linux