linux/drivers/crypto/caam/qi.c

// SPDX-License-Identifier: GPL-2.0
/*
 * CAAM/SEC 4.x QI transport/backend driver
 * Queue Interface backend functionality
 *
 * Copyright 2013-2016 Freescale Semiconductor, Inc.
 * Copyright 2016-2017, 2019-2020 NXP
 */

#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/netdevice.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <soc/fsl/qman.h>

#include "debugfs.h"
#include "regs.h"
#include "qi.h"
#include "desc.h"
#include "intern.h"
#include "desc_constr.h"

#define PREHDR_RSLS_SHIFT …
#define PREHDR_ABS …

/*
 * Use a reasonable backlog of frames (per CPU) as congestion threshold,
 * so that resources used by the in-flight buffers do not become a memory hog.
 */
#define MAX_RSP_FQ_BACKLOG_PER_CPU …

#define CAAM_QI_ENQUEUE_RETRIES …

#define CAAM_NAPI_WEIGHT …

/*
 * caam_napi - struct holding CAAM NAPI-related params
 * @irqtask: IRQ task for QI backend
 * @p: QMan portal
 */
struct caam_napi { … };

/*
 * caam_qi_pcpu_priv - percpu private data structure to main list of pending
 *                     responses expected on each cpu.
 * @caam_napi: CAAM NAPI params
 * @net_dev: netdev used by NAPI
 * @rsp_fq: response FQ from CAAM
 */
struct caam_qi_pcpu_priv { … } ____cacheline_aligned;

static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
static DEFINE_PER_CPU(int, last_cpu);

/*
 * caam_qi_priv - CAAM QI backend private params
 * @cgr: QMan congestion group
 */
struct caam_qi_priv { … };

static struct caam_qi_priv qipriv ____cacheline_aligned;

/*
 * This is written by only one core - the one that initialized the CGR - and
 * read by multiple cores (all the others).
 */
bool caam_congested __read_mostly;
EXPORT_SYMBOL(…);

/*
 * This is a cache of buffers, from which the users of CAAM QI driver
 * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
 * doing malloc on the hotpath.
 * NOTE: A more elegant solution would be to have some headroom in the frames
 *       being processed. This could be added by the dpaa-ethernet driver.
 *       This would pose a problem for userspace application processing which
 *       cannot know of this limitation. So for now, this will work.
 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
 */
static struct kmem_cache *qi_cache;

static void *caam_iova_to_virt(struct iommu_domain *domain,
			       dma_addr_t iova_addr)
{ … }

int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
{ … }
EXPORT_SYMBOL(…);

static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
			   const union qm_mr_entry *msg)
{ … }

static struct qman_fq *create_caam_req_fq(struct device *qidev,
					  struct qman_fq *rsp_fq,
					  dma_addr_t hwdesc,
					  int fq_sched_flag)
{ … }

static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
{ … }

static int kill_fq(struct device *qidev, struct qman_fq *fq)
{ … }

static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx)
{ … }

int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
{ … }
EXPORT_SYMBOL(…);

struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
				       int *cpu,
				       u32 *sh_desc)
{ … }
EXPORT_SYMBOL(…);

void *qi_cache_alloc(gfp_t flags)
{ … }
EXPORT_SYMBOL(…);

void qi_cache_free(void *obj)
{ … }
EXPORT_SYMBOL(…);

static int caam_qi_poll(struct napi_struct *napi, int budget)
{ … }

void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
{ … }
EXPORT_SYMBOL(…);

static void caam_qi_shutdown(void *data)
{ … }

static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
{ … }

static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np,
				 bool sched_napi)
{ … }

static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
						    struct qman_fq *rsp_fq,
						    const struct qm_dqrr_entry *dqrr,
						    bool sched_napi)
{ … }

static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
{ … }

static int init_cgr(struct device *qidev)
{ … }

static int alloc_rsp_fqs(struct device *qidev)
{ … }

static void free_rsp_fqs(void)
{ … }

static void free_caam_qi_pcpu_netdev(const cpumask_t *cpus)
{ … }

int caam_qi_init(struct platform_device *caam_pdev)
{ … }