linux/drivers/infiniband/hw/hfi1/pin_system.c

// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
/*
 * Copyright(c) 2023 - Cornelis Networks, Inc.
 */

#include <linux/types.h>

#include "hfi.h"
#include "common.h"
#include "device.h"
#include "pinning.h"
#include "mmu_rb.h"
#include "user_sdma.h"
#include "trace.h"

struct sdma_mmu_node { … };

static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
			   unsigned long len);
static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, void *arg2,
			 bool *stop);
static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode);

static const struct mmu_rb_ops sdma_rb_ops = …;

int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq)
{ … }

void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq)
{ … }

static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
{ … }

static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
			       unsigned int start, unsigned int npages)
{ … }

static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)
{ … }

static void free_system_node(struct sdma_mmu_node *node)
{ … }

/*
 * kref_get()'s an additional kref on the returned rb_node to prevent rb_node
 * from being released until after rb_node is assigned to an SDMA descriptor
 * (struct sdma_desc) under add_system_iovec_to_sdma_packet(), even if the
 * virtual address range for rb_node is invalidated between now and then.
 */
static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler,
					      unsigned long start,
					      unsigned long end)
{ … }

static int pin_system_pages(struct user_sdma_request *req,
			    uintptr_t start_address, size_t length,
			    struct sdma_mmu_node *node, int npages)
{ … }

/*
 * kref refcount on *node_p will be 2 on successful addition: one kref from
 * kref_init() for mmu_rb_handler and one kref to prevent *node_p from being
 * released until after *node_p is assigned to an SDMA descriptor (struct
 * sdma_desc) under add_system_iovec_to_sdma_packet(), even if the virtual
 * address range for *node_p is invalidated between now and then.
 */
static int add_system_pinning(struct user_sdma_request *req,
			      struct sdma_mmu_node **node_p,
			      unsigned long start, unsigned long len)

{ … }

static int get_system_cache_entry(struct user_sdma_request *req,
				  struct sdma_mmu_node **node_p,
				  size_t req_start, size_t req_len)
{ … }

static void sdma_mmu_rb_node_get(void *ctx)
{ … }

static void sdma_mmu_rb_node_put(void *ctx)
{ … }

static int add_mapping_to_sdma_packet(struct user_sdma_request *req,
				      struct user_sdma_txreq *tx,
				      struct sdma_mmu_node *cache_entry,
				      size_t start,
				      size_t from_this_cache_entry)
{ … }

static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,
					   struct user_sdma_txreq *tx,
					   struct user_sdma_iovec *iovec,
					   size_t from_this_iovec)
{ … }

/*
 * Add up to pkt_data_remaining bytes to the txreq, starting at the current
 * offset in the given iovec entry and continuing until all data has been added
 * to the iovec or the iovec entry type changes.
 *
 * On success, prior to returning, adjust pkt_data_remaining, req->iov_idx, and
 * the offset value in req->iov[req->iov_idx] to reflect the data that has been
 * consumed.
 */
int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req,
				  struct user_sdma_txreq *tx,
				  struct user_sdma_iovec *iovec,
				  u32 *pkt_data_remaining)
{ … }

static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
			   unsigned long len)
{ … }

/*
 * Return 1 to remove the node from the rb tree and call the remove op.
 *
 * Called with the rb tree lock held.
 */
static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,
			 void *evict_arg, bool *stop)
{ … }

static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)
{ … }