linux/drivers/infiniband/ulp/isert/ib_isert.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*******************************************************************************
 * This file contains iSCSI extentions for RDMA (iSER) Verbs
 *
 * (c) Copyright 2013 Datera, Inc.
 *
 * Nicholas A. Bellinger <[email protected]>
 *
 ****************************************************************************/

#include <linux/string.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_cm.h>
#include <rdma/rdma_cm.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
#include <target/iscsi/iscsi_transport.h>
#include <linux/semaphore.h>

#include "ib_isert.h"

static int isert_debug_level;
module_param_named(debug_level, isert_debug_level, int, 0644);
MODULE_PARM_DESC(…) …;

static int isert_sg_tablesize_set(const char *val,
				  const struct kernel_param *kp);
static const struct kernel_param_ops sg_tablesize_ops = …;

static int isert_sg_tablesize = …;
module_param_cb(…);
MODULE_PARM_DESC(…) …;

static DEFINE_MUTEX(device_list_mutex);
static LIST_HEAD(device_list);
static struct workqueue_struct *isert_login_wq;
static struct workqueue_struct *isert_comp_wq;
static struct workqueue_struct *isert_release_wq;

static int
isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd);
static int
isert_login_post_recv(struct isert_conn *isert_conn);
static int
isert_rdma_accept(struct isert_conn *isert_conn);
struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);

static void isert_release_work(struct work_struct *work);
static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);

static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp)
{ … }

static inline bool
isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
{ … }

static void
isert_qp_event_callback(struct ib_event *e, void *context)
{ … }

static struct ib_qp *
isert_create_qp(struct isert_conn *isert_conn,
		struct rdma_cm_id *cma_id)
{ … }

static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{ … }

static void
isert_free_rx_descriptors(struct isert_conn *isert_conn)
{ … }

static int
isert_create_device_ib_res(struct isert_device *device)
{ … }

static void
isert_free_device_ib_res(struct isert_device *device)
{ … }

static void
isert_device_put(struct isert_device *device)
{ … }

static struct isert_device *
isert_device_get(struct rdma_cm_id *cma_id)
{ … }

static void
isert_init_conn(struct isert_conn *isert_conn)
{ … }

static void
isert_free_login_buf(struct isert_conn *isert_conn)
{ … }

static int
isert_alloc_login_buf(struct isert_conn *isert_conn,
		      struct ib_device *ib_dev)
{ … }

static void
isert_set_nego_params(struct isert_conn *isert_conn,
		      struct rdma_conn_param *param)
{ … }

static void
isert_destroy_qp(struct isert_conn *isert_conn)
{ … }

static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{ … }

static void
isert_connect_release(struct isert_conn *isert_conn)
{ … }

static void
isert_connected_handler(struct rdma_cm_id *cma_id)
{ … }

static void
isert_release_kref(struct kref *kref)
{ … }

static void
isert_put_conn(struct isert_conn *isert_conn)
{ … }

static void
isert_handle_unbound_conn(struct isert_conn *isert_conn)
{ … }

/**
 * isert_conn_terminate() - Initiate connection termination
 * @isert_conn: isert connection struct
 *
 * Notes:
 * In case the connection state is BOUND, move state
 * to TEMINATING and start teardown sequence (rdma_disconnect).
 * In case the connection state is UP, complete flush as well.
 *
 * This routine must be called with mutex held. Thus it is
 * safe to call multiple times.
 */
static void
isert_conn_terminate(struct isert_conn *isert_conn)
{ … }

static int
isert_np_cma_handler(struct isert_np *isert_np,
		     enum rdma_cm_event_type event)
{ … }

static int
isert_disconnected_handler(struct rdma_cm_id *cma_id,
			   enum rdma_cm_event_type event)
{ … }

static int
isert_connect_error(struct rdma_cm_id *cma_id)
{ … }

static int
isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{ … }

static int
isert_post_recvm(struct isert_conn *isert_conn, u32 count)
{ … }

static int
isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
{ … }

static int
isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
{ … }

static void
__isert_create_send_desc(struct isert_device *device,
			 struct iser_tx_desc *tx_desc)
{ … }

static void
isert_create_send_desc(struct isert_conn *isert_conn,
		       struct isert_cmd *isert_cmd,
		       struct iser_tx_desc *tx_desc)
{ … }

static int
isert_init_tx_hdrs(struct isert_conn *isert_conn,
		   struct iser_tx_desc *tx_desc)
{ … }

static void
isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
		   struct ib_send_wr *send_wr)
{ … }

static int
isert_login_post_recv(struct isert_conn *isert_conn)
{ … }

static int
isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login,
		   u32 length)
{ … }

static void
isert_rx_login_req(struct isert_conn *isert_conn)
{ … }

static struct iscsit_cmd
*isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc)
{ … }

static int
isert_handle_scsi_cmd(struct isert_conn *isert_conn,
		      struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd,
		      struct iser_rx_desc *rx_desc, unsigned char *buf)
{ … }

static int
isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
			   struct iser_rx_desc *rx_desc, unsigned char *buf)
{ … }

static int
isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
		     struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc,
		     unsigned char *buf)
{ … }

static int
isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
		      struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc,
		      struct iscsi_text *hdr)
{ … }

static int
isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
		uint32_t read_stag, uint64_t read_va,
		uint32_t write_stag, uint64_t write_va)
{ … }

static void
isert_print_wc(struct ib_wc *wc, const char *type)
{ … }

static void
isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static void
isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static void
isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
{ … }

static void
isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
{ … }

static void
isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
{ … }

static void
isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
		     struct ib_device *ib_dev, bool comp_err)
{ … }

static int
isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
{ … }

static void
isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static void
isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static void
isert_do_control_comp(struct work_struct *work)
{ … }

static void
isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static void
isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

static int
isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
{ … }

static int
isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
{ … }

static void
isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
{ … }

static enum target_prot_op
isert_get_sup_prot_ops(struct iscsit_conn *conn)
{ … }

static int
isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn,
		bool nopout_response)
{ … }

static int
isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
{ … }

static int
isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
{ … }

static int
isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
{ … }

static int
isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
{ … }

static inline void
isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain)
{ … }

static int
isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
{ … }

static int
isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
		struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
{ … }

static int
isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
{ … }

static int
isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery)
{ … }

static int
isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state)
{ … }

static int
isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state)
{ … }

struct rdma_cm_id *
isert_setup_id(struct isert_np *isert_np)
{ … }

static int
isert_setup_np(struct iscsi_np *np,
	       struct sockaddr_storage *ksockaddr)
{ … }

static int
isert_rdma_accept(struct isert_conn *isert_conn)
{ … }

static int
isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login)
{ … }

static void
isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn,
		    struct isert_conn *isert_conn)
{ … }

static int
isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn)
{ … }

static void
isert_free_np(struct iscsi_np *np)
{ … }

static void isert_release_work(struct work_struct *work)
{ … }

static void
isert_wait4logout(struct isert_conn *isert_conn)
{ … }

static void
isert_wait4cmds(struct iscsit_conn *conn)
{ … }

/**
 * isert_put_unsol_pending_cmds() - Drop commands waiting for
 *     unsolicitate dataout
 * @conn:    iscsi connection
 *
 * We might still have commands that are waiting for unsolicited
 * dataouts messages. We must put the extra reference on those
 * before blocking on the target_wait_for_session_cmds
 */
static void
isert_put_unsol_pending_cmds(struct iscsit_conn *conn)
{ … }

static void isert_wait_conn(struct iscsit_conn *conn)
{ … }

static void isert_free_conn(struct iscsit_conn *conn)
{ … }

static void isert_get_rx_pdu(struct iscsit_conn *conn)
{ … }

static struct iscsit_transport iser_target_transport = …;

static int __init isert_init(void)
{ … }

static void __exit isert_exit(void)
{ … }

MODULE_DESCRIPTION(…) …;
MODULE_AUTHOR(…) …;
MODULE_LICENSE(…) …;

module_init(…) …;
module_exit(isert_exit);