linux/net/smc/smc_tx.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Manage send buffer.
 * Producer:
 * Copy user space data into send buffer, if send buffer space available.
 * Consumer:
 * Trigger RDMA write into RMBE of peer and send CDC, if RMBE space available.
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Ursula Braun <[email protected]>
 */

#include <linux/net.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/tcp.h>

#include "smc.h"
#include "smc_wr.h"
#include "smc_cdc.h"
#include "smc_close.h"
#include "smc_ism.h"
#include "smc_tx.h"
#include "smc_stats.h"
#include "smc_tracepoint.h"

#define SMC_TX_WORK_DELAY …

/***************************** sndbuf producer *******************************/

/* callback implementation for sk.sk_write_space()
 * to wakeup sndbuf producers that blocked with smc_tx_wait().
 * called under sk_socket lock.
 */
static void smc_tx_write_space(struct sock *sk)
{ … }

/* Wakeup sndbuf producers that blocked with smc_tx_wait().
 * Cf. tcp_data_snd_check()=>tcp_check_space()=>tcp_new_space().
 */
void smc_tx_sndbuf_nonfull(struct smc_sock *smc)
{ … }

/* blocks sndbuf producer until at least one byte of free space available
 * or urgent Byte was consumed
 */
static int smc_tx_wait(struct smc_sock *smc, int flags)
{ … }

static bool smc_tx_is_corked(struct smc_sock *smc)
{ … }

/* If we have pending CDC messages, do not send:
 * Because CQE of this CDC message will happen shortly, it gives
 * a chance to coalesce future sendmsg() payload in to one RDMA Write,
 * without need for a timer, and with no latency trade off.
 * Algorithm here:
 *  1. First message should never cork
 *  2. If we have pending Tx CDC messages, wait for the first CDC
 *     message's completion
 *  3. Don't cork to much data in a single RDMA Write to prevent burst
 *     traffic, total corked message should not exceed sendbuf/2
 */
static bool smc_should_autocork(struct smc_sock *smc)
{ … }

static bool smc_tx_should_cork(struct smc_sock *smc, struct msghdr *msg)
{ … }

/* sndbuf producer: main API called by socket layer.
 * called under sock lock.
 */
int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
{ … }

/***************************** sndbuf consumer *******************************/

/* sndbuf consumer: actual data transfer of one target chunk with ISM write */
int smcd_tx_ism_write(struct smc_connection *conn, void *data, size_t len,
		      u32 offset, int signal)
{ … }

/* sndbuf consumer: actual data transfer of one target chunk with RDMA write */
static int smc_tx_rdma_write(struct smc_connection *conn, int peer_rmbe_offset,
			     int num_sges, struct ib_rdma_wr *rdma_wr)
{ … }

/* sndbuf consumer */
static inline void smc_tx_advance_cursors(struct smc_connection *conn,
					  union smc_host_cursor *prod,
					  union smc_host_cursor *sent,
					  size_t len)
{ … }

/* SMC-R helper for smc_tx_rdma_writes() */
static int smcr_tx_rdma_writes(struct smc_connection *conn, size_t len,
			       size_t src_off, size_t src_len,
			       size_t dst_off, size_t dst_len,
			       struct smc_rdma_wr *wr_rdma_buf)
{ … }

/* SMC-D helper for smc_tx_rdma_writes() */
static int smcd_tx_rdma_writes(struct smc_connection *conn, size_t len,
			       size_t src_off, size_t src_len,
			       size_t dst_off, size_t dst_len)
{ … }

/* sndbuf consumer: prepare all necessary (src&dst) chunks of data transmit;
 * usable snd_wnd as max transmit
 */
static int smc_tx_rdma_writes(struct smc_connection *conn,
			      struct smc_rdma_wr *wr_rdma_buf)
{ … }

/* Wakeup sndbuf consumers from any context (IRQ or process)
 * since there is more data to transmit; usable snd_wnd as max transmit
 */
static int smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
{ … }

static int smcd_tx_sndbuf_nonempty(struct smc_connection *conn)
{ … }

int smc_tx_sndbuf_nonempty(struct smc_connection *conn)
{ … }

/* Wakeup sndbuf consumers from process context
 * since there is more data to transmit. The caller
 * must hold sock lock.
 */
void smc_tx_pending(struct smc_connection *conn)
{ … }

/* Wakeup sndbuf consumers from process context
 * since there is more data to transmit in locked
 * sock.
 */
void smc_tx_work(struct work_struct *work)
{ … }

void smc_tx_consumer_update(struct smc_connection *conn, bool force)
{ … }

/***************************** send initialize *******************************/

/* Initialize send properties on connection establishment. NB: not __init! */
void smc_tx_init(struct smc_sock *smc)
{ … }