pci.c | Explore in Territory

// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
/* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/export.h>
#include <linux/err.h>
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
#include <linux/log2.h>
#include <linux/string.h>
#include <net/page_pool/helpers.h>

#include "pci_hw.h"
#include "pci.h"
#include "core.h"
#include "cmd.h"
#include "port.h"
#include "resources.h"

#define mlxsw_pci_write32(mlxsw_pci, reg, val) …
#define mlxsw_pci_read32(mlxsw_pci, reg) …

enum mlxsw_pci_queue_type { … };

#define MLXSW_PCI_QUEUE_TYPE_COUNT …

enum mlxsw_pci_cq_type { … };

static const u16 mlxsw_pci_doorbell_type_offset[] = …;

static const u16 mlxsw_pci_doorbell_arm_type_offset[] = …;

struct mlxsw_pci_mem_item { … };

struct mlxsw_pci_queue_elem_info { … };

struct mlxsw_pci_queue { … };

struct mlxsw_pci_queue_type_group { … };

struct mlxsw_pci { … };

static int mlxsw_pci_napi_devs_init(struct mlxsw_pci *mlxsw_pci)
{ … }

static void mlxsw_pci_napi_devs_fini(struct mlxsw_pci *mlxsw_pci)
{ … }

static char *__mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q,
					size_t elem_size, int elem_index)
{ … }

static struct mlxsw_pci_queue_elem_info *
mlxsw_pci_queue_elem_info_get(struct mlxsw_pci_queue *q, int elem_index)
{ … }

static struct mlxsw_pci_queue_elem_info *
mlxsw_pci_queue_elem_info_producer_get(struct mlxsw_pci_queue *q)
{ … }

static struct mlxsw_pci_queue_elem_info *
mlxsw_pci_queue_elem_info_consumer_get(struct mlxsw_pci_queue *q)
{ … }

static char *mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q, int elem_index)
{ … }

static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)
{ … }

static struct mlxsw_pci_queue_type_group *
mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,
			       enum mlxsw_pci_queue_type q_type)
{ … }

static struct mlxsw_pci_queue *
__mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci,
		      enum mlxsw_pci_queue_type q_type, u8 q_num)
{ … }

static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci,
						 u8 q_num)
{ … }

static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci,
						u8 q_num)
{ … }

static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci)
{ … }

static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci,
					   struct mlxsw_pci_queue *q,
					   u16 val)
{ … }

static void __mlxsw_pci_queue_doorbell_arm_set(struct mlxsw_pci *mlxsw_pci,
					       struct mlxsw_pci_queue *q,
					       u16 val)
{ … }

static void mlxsw_pci_queue_doorbell_producer_ring(struct mlxsw_pci *mlxsw_pci,
						   struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_queue_doorbell_consumer_ring(struct mlxsw_pci *mlxsw_pci,
						   struct mlxsw_pci_queue *q)
{ … }

static void
mlxsw_pci_queue_doorbell_arm_consumer_ring(struct mlxsw_pci *mlxsw_pci,
					   struct mlxsw_pci_queue *q)
{ … }

static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q,
					     int page_index)
{ … }

static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
			      struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_sdq_fini(struct mlxsw_pci *mlxsw_pci,
			       struct mlxsw_pci_queue *q)
{ … }

#define MLXSW_PCI_SKB_HEADROOM …

#define MLXSW_PCI_RX_BUF_SW_OVERHEAD …

static void
mlxsw_pci_wqe_rx_frag_set(struct mlxsw_pci *mlxsw_pci, struct page *page,
			  char *wqe, int index, size_t frag_len)
{ … }

static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe,
				  int index, char *frag_data, size_t frag_len,
				  int direction)
{ … }

static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe,
				     int index, int direction)
{ … }

static struct sk_buff *mlxsw_pci_rdq_build_skb(struct mlxsw_pci_queue *q,
					       struct page *pages[],
					       u16 byte_count)
{ … }

static int mlxsw_pci_rdq_page_alloc(struct mlxsw_pci_queue *q,
				    struct mlxsw_pci_queue_elem_info *elem_info,
				    int index)
{ … }

static void mlxsw_pci_rdq_page_free(struct mlxsw_pci_queue *q,
				    struct mlxsw_pci_queue_elem_info *elem_info,
				    int index)
{ … }

static u8 mlxsw_pci_num_sg_entries_get(u16 byte_count)
{ … }

static int
mlxsw_pci_elem_info_pages_ref_store(const struct mlxsw_pci_queue *q,
				    const struct mlxsw_pci_queue_elem_info *el,
				    u16 byte_count, struct page *pages[],
				    u8 *p_num_sg_entries)
{ … }

static int
mlxsw_pci_rdq_pages_alloc(struct mlxsw_pci_queue *q,
			  struct mlxsw_pci_queue_elem_info *elem_info,
			  u8 num_sg_entries)
{ … }

static void
mlxsw_pci_rdq_pages_recycle(struct mlxsw_pci_queue *q, struct page *pages[],
			    u8 num_sg_entries)
{ … }

static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
			      struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,
			       struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci,
				  struct mlxsw_pci_queue *q)
{ … }

static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci,
					 ptrdiff_t off)
{ … }

static void mlxsw_pci_skb_cb_ts_set(struct mlxsw_pci *mlxsw_pci,
				    struct sk_buff *skb,
				    enum mlxsw_pci_cqe_v cqe_v, char *cqe)
{ … }

static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,
				     struct mlxsw_pci_queue *q,
				     u16 consumer_counter_limit,
				     enum mlxsw_pci_cqe_v cqe_v,
				     char *cqe, int budget)
{ … }

static void mlxsw_pci_cqe_rdq_md_tx_port_init(struct sk_buff *skb,
					      const char *cqe)
{ … }

static void mlxsw_pci_cqe_rdq_md_init(struct sk_buff *skb, const char *cqe)
{ … }

static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
				     struct mlxsw_pci_queue *q,
				     u16 consumer_counter_limit,
				     enum mlxsw_pci_cqe_v cqe_v, char *cqe)
{ … }

static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)
{ … }

static bool mlxsw_pci_cq_cqe_to_handle(struct mlxsw_pci_queue *q)
{ … }

static int mlxsw_pci_napi_poll_cq_rx(struct napi_struct *napi, int budget)
{ … }

static int mlxsw_pci_napi_poll_cq_tx(struct napi_struct *napi, int budget)
{ … }

static enum mlxsw_pci_cq_type
mlxsw_pci_cq_type(const struct mlxsw_pci *mlxsw_pci,
		  const struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_cq_napi_setup(struct mlxsw_pci_queue *q,
				    enum mlxsw_pci_cq_type cq_type)
{ … }

static void mlxsw_pci_cq_napi_teardown(struct mlxsw_pci_queue *q)
{ … }

static int mlxsw_pci_cq_page_pool_init(struct mlxsw_pci_queue *q,
				       enum mlxsw_pci_cq_type cq_type)
{ … }

static void mlxsw_pci_cq_page_pool_fini(struct mlxsw_pci_queue *q,
					enum mlxsw_pci_cq_type cq_type)
{ … }

static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
			     struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci,
			      struct mlxsw_pci_queue *q)
{ … }

static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q)
{ … }

static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q)
{ … }

static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t)
{ … }

static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
			     struct mlxsw_pci_queue *q)
{ … }

static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci,
			      struct mlxsw_pci_queue *q)
{ … }

struct mlxsw_pci_queue_ops { … };

static const struct mlxsw_pci_queue_ops mlxsw_pci_sdq_ops = …;

static const struct mlxsw_pci_queue_ops mlxsw_pci_rdq_ops = …;

static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = …;

static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = …;

static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
				const struct mlxsw_pci_queue_ops *q_ops,
				struct mlxsw_pci_queue *q, u8 q_num)
{ … }

static void mlxsw_pci_queue_fini(struct mlxsw_pci *mlxsw_pci,
				 const struct mlxsw_pci_queue_ops *q_ops,
				 struct mlxsw_pci_queue *q)
{ … }

static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
				      const struct mlxsw_pci_queue_ops *q_ops,
				      u8 num_qs)
{ … }

static void mlxsw_pci_queue_group_fini(struct mlxsw_pci *mlxsw_pci,
				       const struct mlxsw_pci_queue_ops *q_ops)
{ … }

static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
{ … }

static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci)
{ … }

static void
mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,
				     char *mbox, int index,
				     const struct mlxsw_swid_config *swid)
{ … }

static int
mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_pci *mlxsw_pci,
				const struct mlxsw_config_profile *profile,
				struct mlxsw_res *res)
{ … }

static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
				    const struct mlxsw_config_profile *profile,
				    struct mlxsw_res *res)
{ … }

static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)
{ … }

static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
				  u16 num_pages)
{ … }

static void mlxsw_pci_fw_area_fini(struct mlxsw_pci *mlxsw_pci)
{ … }

static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id)
{ … }

static int mlxsw_pci_mbox_alloc(struct mlxsw_pci *mlxsw_pci,
				struct mlxsw_pci_mem_item *mbox)
{ … }

static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,
				struct mlxsw_pci_mem_item *mbox)
{ … }

static int mlxsw_pci_sys_ready_wait(struct mlxsw_pci *mlxsw_pci,
				    const struct pci_device_id *id,
				    u32 *p_sys_status)
{ … }

static int mlxsw_pci_reset_at_pci_disable(struct mlxsw_pci *mlxsw_pci,
					  bool pci_reset_sbr_supported)
{ … }

static int mlxsw_pci_reset_sw(struct mlxsw_pci *mlxsw_pci)
{ … }

static int
mlxsw_pci_reset(struct mlxsw_pci *mlxsw_pci, const struct pci_device_id *id)
{ … }

static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)
{ … }

static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)
{ … }

static void mlxsw_pci_num_sg_entries_set(struct mlxsw_pci *mlxsw_pci)
{ … }

static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
			  const struct mlxsw_config_profile *profile,
			  struct mlxsw_res *res)
{ … }

static void mlxsw_pci_fini(void *bus_priv)
{ … }

static struct mlxsw_pci_queue *
mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci,
		   const struct mlxsw_tx_info *tx_info)
{ … }

static bool mlxsw_pci_skb_transmit_busy(void *bus_priv,
					const struct mlxsw_tx_info *tx_info)
{ … }

static int mlxsw_pci_skb_transmit(void *bus_priv, struct sk_buff *skb,
				  const struct mlxsw_tx_info *tx_info)
{ … }

static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
			      u32 in_mod, bool out_mbox_direct,
			      char *in_mbox, size_t in_mbox_size,
			      char *out_mbox, size_t out_mbox_size,
			      u8 *p_status)
{ … }

static u32 mlxsw_pci_read_frc_h(void *bus_priv)
{ … }

static u32 mlxsw_pci_read_frc_l(void *bus_priv)
{ … }

static u32 mlxsw_pci_read_utc_sec(void *bus_priv)
{ … }

static u32 mlxsw_pci_read_utc_nsec(void *bus_priv)
{ … }

static enum mlxsw_cmd_mbox_config_profile_lag_mode
mlxsw_pci_lag_mode(void *bus_priv)
{ … }

static enum mlxsw_cmd_mbox_config_profile_flood_mode
mlxsw_pci_flood_mode(void *bus_priv)
{ … }

static const struct mlxsw_bus mlxsw_pci_bus = …;

static int mlxsw_pci_cmd_init(struct mlxsw_pci *mlxsw_pci)
{ … }

static void mlxsw_pci_cmd_fini(struct mlxsw_pci *mlxsw_pci)
{ … }

static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{ … }

static void mlxsw_pci_remove(struct pci_dev *pdev)
{ … }

static void mlxsw_pci_reset_prepare(struct pci_dev *pdev)
{ … }

static void mlxsw_pci_reset_done(struct pci_dev *pdev)
{ … }

static const struct pci_error_handlers mlxsw_pci_err_handler = …;

int mlxsw_pci_driver_register(struct pci_driver *pci_driver)
{ … }
EXPORT_SYMBOL(…);

void mlxsw_pci_driver_unregister(struct pci_driver *pci_driver)
{ … }
EXPORT_SYMBOL(…);

static int __init mlxsw_pci_module_init(void)
{ … }

static void __exit mlxsw_pci_module_exit(void)
{ … }

module_init(…) …;
module_exit(mlxsw_pci_module_exit);

MODULE_LICENSE(…) …;
MODULE_AUTHOR(…) …;
MODULE_DESCRIPTION(…) …;
linux/drivers/net/ethernet/mellanox/mlxsw/pci.c
