linux/net/smc/smc_wr.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Work Requests exploiting Infiniband API
 *
 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
 * are submitted to either RC SQ or RC RQ respectively
 * (reliably connected send/receive queue)
 * and become work queue entries (WQEs).
 * While an SQ WR/WQE is pending, we track it until transmission completion.
 * Through a send or receive completion queue (CQ) respectively,
 * we get completion queue entries (CQEs) [aka work completions (WCs)].
 * Since the CQ callback is called from IRQ context, we split work by using
 * bottom halves implemented by tasklets.
 *
 * SMC uses this to exchange LLC (link layer control)
 * and CDC (connection data control) messages.
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Steffen Maier <[email protected]>
 */

#include <linux/atomic.h>
#include <linux/hashtable.h>
#include <linux/wait.h>
#include <rdma/ib_verbs.h>
#include <asm/div64.h>

#include "smc.h"
#include "smc_wr.h"

#define SMC_WR_MAX_POLL_CQE …

#define SMC_WR_RX_HASH_BITS …
static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);

struct smc_wr_tx_pend { … };

/******************************** send queue *********************************/

/*------------------------------- completion --------------------------------*/

/* returns true if at least one tx work request is pending on the given link */
static inline bool smc_wr_is_tx_pend(struct smc_link *link)
{ … }

/* wait till all pending tx work requests on the given link are completed */
void smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
{ … }

static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
{ … }

static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
{ … }

static void smc_wr_tx_tasklet_fn(struct tasklet_struct *t)
{ … }

void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
{ … }

/*---------------------------- request submission ---------------------------*/

static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
{ … }

/**
 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
 *			and sets info for pending transmit tracking
 * @link:		Pointer to smc_link used to later send the message.
 * @handler:		Send completion handler function pointer.
 * @wr_buf:		Out value returns pointer to message buffer.
 * @wr_rdma_buf:	Out value returns pointer to rdma work request.
 * @wr_pend_priv:	Out value returns pointer serving as handler context.
 *
 * Return: 0 on success, or -errno on error.
 */
int smc_wr_tx_get_free_slot(struct smc_link *link,
			    smc_wr_tx_handler handler,
			    struct smc_wr_buf **wr_buf,
			    struct smc_rdma_wr **wr_rdma_buf,
			    struct smc_wr_tx_pend_priv **wr_pend_priv)
{ … }

int smc_wr_tx_get_v2_slot(struct smc_link *link,
			  smc_wr_tx_handler handler,
			  struct smc_wr_v2_buf **wr_buf,
			  struct smc_wr_tx_pend_priv **wr_pend_priv)
{ … }

int smc_wr_tx_put_slot(struct smc_link *link,
		       struct smc_wr_tx_pend_priv *wr_pend_priv)
{ … }

/* Send prepared WR slot via ib_post_send.
 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
 */
int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
{ … }

int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
		      int len)
{ … }

/* Send prepared WR slot via ib_post_send and wait for send completion
 * notification.
 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
 */
int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
			unsigned long timeout)
{ … }

/* Register a memory region and wait for result. */
int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
{ … }

/****************************** receive queue ********************************/

int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
{ … }

/* Demultiplex a received work request based on the message type to its handler.
 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
 * and not being modified any more afterwards so we don't need to lock it.
 */
static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
{ … }

static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
{ … }

static void smc_wr_rx_tasklet_fn(struct tasklet_struct *t)
{ … }

void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
{ … }

int smc_wr_rx_post_init(struct smc_link *link)
{ … }

/***************************** init, exit, misc ******************************/

void smc_wr_remember_qp_attr(struct smc_link *lnk)
{ … }

static void smc_wr_init_sge(struct smc_link *lnk)
{ … }

void smc_wr_free_link(struct smc_link *lnk)
{ … }

void smc_wr_free_lgr_mem(struct smc_link_group *lgr)
{ … }

void smc_wr_free_link_mem(struct smc_link *lnk)
{ … }

int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr)
{ … }

int smc_wr_alloc_link_mem(struct smc_link *link)
{ … }

void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
{ … }

void smc_wr_add_dev(struct smc_ib_device *smcibdev)
{ … }

static void smcr_wr_tx_refs_free(struct percpu_ref *ref)
{ … }

static void smcr_wr_reg_refs_free(struct percpu_ref *ref)
{ … }

int smc_wr_create_link(struct smc_link *lnk)
{ … }