// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) /* Copyright (C) 2015-2019 Netronome Systems, Inc. */ /* * nfp_net_common.c * Netronome network device driver: Common functions between PF and VF * Authors: Jakub Kicinski <[email protected]> * Jason McMullan <[email protected]> * Rolf Neugebauer <[email protected]> * Brad Petrus <[email protected]> * Chris Telfer <[email protected]> */ #include <linux/bitfield.h> #include <linux/bpf.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/interrupt.h> #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/mm.h> #include <linux/overflow.h> #include <linux/page_ref.h> #include <linux/pci.h> #include <linux/pci_regs.h> #include <linux/ethtool.h> #include <linux/log2.h> #include <linux/if_vlan.h> #include <linux/if_bridge.h> #include <linux/random.h> #include <linux/vmalloc.h> #include <linux/ktime.h> #include <net/tls.h> #include <net/vxlan.h> #include <net/xdp_sock_drv.h> #include <net/xfrm.h> #include "nfpcore/nfp_dev.h" #include "nfpcore/nfp_nsp.h" #include "ccm.h" #include "nfp_app.h" #include "nfp_net_ctrl.h" #include "nfp_net.h" #include "nfp_net_dp.h" #include "nfp_net_sriov.h" #include "nfp_net_xsk.h" #include "nfp_port.h" #include "crypto/crypto.h" #include "crypto/fw.h" static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr); /** * nfp_net_get_fw_version() - Read and parse the FW version * @fw_ver: Output fw_version structure to read to * @ctrl_bar: Mapped address of the control BAR */ void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver, void __iomem *ctrl_bar) { … } u32 nfp_qcp_queue_offset(const struct nfp_dev_info *dev_info, u16 queue) { … } /* Firmware reconfig * * Firmware reconfig may take a while so we have two versions of it - * synchronous and asynchronous (posted). All synchronous callers are holding * RTNL so we don't have to worry about serializing them. */ static void nfp_net_reconfig_start(struct nfp_net *nn, u32 update) { … } /* Pass 0 as update to run posted reconfigs. */ static void nfp_net_reconfig_start_async(struct nfp_net *nn, u32 update) { … } static bool nfp_net_reconfig_check_done(struct nfp_net *nn, bool last_check) { … } static bool __nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline) { … } static int nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline) { … } static void nfp_net_reconfig_timer(struct timer_list *t) { … } /** * nfp_net_reconfig_post() - Post async reconfig request * @nn: NFP Net device to reconfigure * @update: The value for the update field in the BAR config * * Record FW reconfiguration request. Reconfiguration will be kicked off * whenever reconfiguration machinery is idle. Multiple requests can be * merged together! */ static void nfp_net_reconfig_post(struct nfp_net *nn, u32 update) { … } static void nfp_net_reconfig_sync_enter(struct nfp_net *nn) { … } static void nfp_net_reconfig_wait_posted(struct nfp_net *nn) { … } /** * __nfp_net_reconfig() - Reconfigure the firmware * @nn: NFP Net device to reconfigure * @update: The value for the update field in the BAR config * * Write the update word to the BAR and ping the reconfig queue. The * poll until the firmware has acknowledged the update by zeroing the * update word. * * Return: Negative errno on error, 0 on success */ int __nfp_net_reconfig(struct nfp_net *nn, u32 update) { … } int nfp_net_reconfig(struct nfp_net *nn, u32 update) { … } int nfp_net_mbox_lock(struct nfp_net *nn, unsigned int data_size) { … } /** * nfp_net_mbox_reconfig() - Reconfigure the firmware via the mailbox * @nn: NFP Net device to reconfigure * @mbox_cmd: The value for the mailbox command * * Helper function for mailbox updates * * Return: Negative errno on error, 0 on success */ int nfp_net_mbox_reconfig(struct nfp_net *nn, u32 mbox_cmd) { … } void nfp_net_mbox_reconfig_post(struct nfp_net *nn, u32 mbox_cmd) { … } int nfp_net_mbox_reconfig_wait_posted(struct nfp_net *nn) { … } int nfp_net_mbox_reconfig_and_unlock(struct nfp_net *nn, u32 mbox_cmd) { … } /* Interrupt configuration and handling */ /** * nfp_net_irqs_alloc() - allocates MSI-X irqs * @pdev: PCI device structure * @irq_entries: Array to be initialized and used to hold the irq entries * @min_irqs: Minimal acceptable number of interrupts * @wanted_irqs: Target number of interrupts to allocate * * Return: Number of irqs obtained or 0 on error. */ unsigned int nfp_net_irqs_alloc(struct pci_dev *pdev, struct msix_entry *irq_entries, unsigned int min_irqs, unsigned int wanted_irqs) { … } /** * nfp_net_irqs_assign() - Assign interrupts allocated externally to netdev * @nn: NFP Network structure * @irq_entries: Table of allocated interrupts * @n: Size of @irq_entries (number of entries to grab) * * After interrupts are allocated with nfp_net_irqs_alloc() this function * should be called to assign them to a specific netdev (port). */ void nfp_net_irqs_assign(struct nfp_net *nn, struct msix_entry *irq_entries, unsigned int n) { … } /** * nfp_net_irqs_disable() - Disable interrupts * @pdev: PCI device structure * * Undoes what @nfp_net_irqs_alloc() does. */ void nfp_net_irqs_disable(struct pci_dev *pdev) { … } /** * nfp_net_irq_rxtx() - Interrupt service routine for RX/TX rings. * @irq: Interrupt * @data: Opaque data structure * * Return: Indicate if the interrupt has been handled. */ static irqreturn_t nfp_net_irq_rxtx(int irq, void *data) { … } static irqreturn_t nfp_ctrl_irq_rxtx(int irq, void *data) { … } /** * nfp_net_read_link_status() - Reread link status from control BAR * @nn: NFP Network structure */ static void nfp_net_read_link_status(struct nfp_net *nn) { … } /** * nfp_net_irq_lsc() - Interrupt service routine for link state changes * @irq: Interrupt * @data: Opaque data structure * * Return: Indicate if the interrupt has been handled. */ static irqreturn_t nfp_net_irq_lsc(int irq, void *data) { … } /** * nfp_net_irq_exn() - Interrupt service routine for exceptions * @irq: Interrupt * @data: Opaque data structure * * Return: Indicate if the interrupt has been handled. */ static irqreturn_t nfp_net_irq_exn(int irq, void *data) { … } /** * nfp_net_aux_irq_request() - Request an auxiliary interrupt (LSC or EXN) * @nn: NFP Network structure * @ctrl_offset: Control BAR offset where IRQ configuration should be written * @format: printf-style format to construct the interrupt name * @name: Pointer to allocated space for interrupt name * @name_sz: Size of space for interrupt name * @vector_idx: Index of MSI-X vector used for this interrupt * @handler: IRQ handler to register for this interrupt */ static int nfp_net_aux_irq_request(struct nfp_net *nn, u32 ctrl_offset, const char *format, char *name, size_t name_sz, unsigned int vector_idx, irq_handler_t handler) { … } /** * nfp_net_aux_irq_free() - Free an auxiliary interrupt (LSC or EXN) * @nn: NFP Network structure * @ctrl_offset: Control BAR offset where IRQ configuration should be written * @vector_idx: Index of MSI-X vector used for this interrupt */ static void nfp_net_aux_irq_free(struct nfp_net *nn, u32 ctrl_offset, unsigned int vector_idx) { … } struct sk_buff * nfp_net_tls_tx(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, struct sk_buff *skb, u64 *tls_handle, int *nr_frags) { … } void nfp_net_tls_tx_undo(struct sk_buff *skb, u64 tls_handle) { … } static void nfp_net_tx_timeout(struct net_device *netdev, unsigned int txqueue) { … } /* Receive processing */ static unsigned int nfp_net_calc_fl_bufsz_data(struct nfp_net_dp *dp) { … } static unsigned int nfp_net_calc_fl_bufsz(struct nfp_net_dp *dp) { … } static unsigned int nfp_net_calc_fl_bufsz_xsk(struct nfp_net_dp *dp) { … } /* Setup and Configuration */ /** * nfp_net_vecs_init() - Assign IRQs and setup rvecs. * @nn: NFP Network structure */ static void nfp_net_vecs_init(struct nfp_net *nn) { … } static void nfp_net_napi_add(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx) { … } static void nfp_net_napi_del(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec) { … } static void nfp_net_vector_assign_rings(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx) { … } static int nfp_net_prepare_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec, int idx) { … } static void nfp_net_cleanup_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec) { … } /** * nfp_net_rss_write_itbl() - Write RSS indirection table to device * @nn: NFP Net device to reconfigure */ void nfp_net_rss_write_itbl(struct nfp_net *nn) { … } /** * nfp_net_rss_write_key() - Write RSS hash key to device * @nn: NFP Net device to reconfigure */ void nfp_net_rss_write_key(struct nfp_net *nn) { … } /** * nfp_net_coalesce_write_cfg() - Write irq coalescence configuration to HW * @nn: NFP Net device to reconfigure */ void nfp_net_coalesce_write_cfg(struct nfp_net *nn) { … } /** * nfp_net_write_mac_addr() - Write mac address to the device control BAR * @nn: NFP Net device to reconfigure * @addr: MAC address to write * * Writes the MAC address from the netdev to the device control BAR. Does not * perform the required reconfig. We do a bit of byte swapping dance because * firmware is LE. */ static void nfp_net_write_mac_addr(struct nfp_net *nn, const u8 *addr) { … } /** * nfp_net_clear_config_and_disable() - Clear control BAR and disable NFP * @nn: NFP Net device to reconfigure * * Warning: must be fully idempotent. */ static void nfp_net_clear_config_and_disable(struct nfp_net *nn) { … } /** * nfp_net_set_config_and_enable() - Write control BAR and enable NFP * @nn: NFP Net device to reconfigure */ static int nfp_net_set_config_and_enable(struct nfp_net *nn) { … } /** * nfp_net_close_stack() - Quiesce the stack (part of close) * @nn: NFP Net device to reconfigure */ static void nfp_net_close_stack(struct nfp_net *nn) { … } /** * nfp_net_close_free_all() - Free all runtime resources * @nn: NFP Net device to reconfigure */ static void nfp_net_close_free_all(struct nfp_net *nn) { … } /** * nfp_net_netdev_close() - Called when the device is downed * @netdev: netdev structure */ static int nfp_net_netdev_close(struct net_device *netdev) { … } void nfp_ctrl_close(struct nfp_net *nn) { … } static void nfp_net_rx_dim_work(struct work_struct *work) { … } static void nfp_net_tx_dim_work(struct work_struct *work) { … } /** * nfp_net_open_stack() - Start the device from stack's perspective * @nn: NFP Net device to reconfigure */ static void nfp_net_open_stack(struct nfp_net *nn) { … } static int nfp_net_open_alloc_all(struct nfp_net *nn) { … } static int nfp_net_netdev_open(struct net_device *netdev) { … } int nfp_ctrl_open(struct nfp_net *nn) { … } int nfp_net_sched_mbox_amsg_work(struct nfp_net *nn, u32 cmd, const void *data, size_t len, int (*cb)(struct nfp_net *, struct nfp_mbox_amsg_entry *)) { … } static void nfp_net_mbox_amsg_work(struct work_struct *work) { … } static int nfp_net_mc_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry) { … } static int nfp_net_mc_sync(struct net_device *netdev, const unsigned char *addr) { … } static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr) { … } static void nfp_net_set_rx_mode(struct net_device *netdev) { … } static void nfp_net_rss_init_itbl(struct nfp_net *nn) { … } static void nfp_net_dp_swap(struct nfp_net *nn, struct nfp_net_dp *dp) { … } static int nfp_net_dp_swap_enable(struct nfp_net *nn, struct nfp_net_dp *dp) { … } struct nfp_net_dp *nfp_net_clone_dp(struct nfp_net *nn) { … } static void nfp_net_free_dp(struct nfp_net_dp *dp) { … } static int nfp_net_check_config(struct nfp_net *nn, struct nfp_net_dp *dp, struct netlink_ext_ack *extack) { … } int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *dp, struct netlink_ext_ack *extack) { … } static int nfp_net_change_mtu(struct net_device *netdev, int new_mtu) { … } static int nfp_net_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) { … } static int nfp_net_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) { … } static void nfp_net_fs_fill_v4(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr) { … } static void nfp_net_fs_fill_v6(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr) { … } #define NFP_FS_QUEUE_ID … #define NFP_FS_ACT … #define NFP_FS_ACT_DROP … #define NFP_FS_ACT_Q … static void nfp_net_fs_fill_act(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 addr) { … } int nfp_net_fs_add_hw(struct nfp_net *nn, struct nfp_fs_entry *entry) { … } int nfp_net_fs_del_hw(struct nfp_net *nn, struct nfp_fs_entry *entry) { … } static void nfp_net_fs_clean(struct nfp_net *nn) { … } static void nfp_net_stat64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { … } static int nfp_net_set_features(struct net_device *netdev, netdev_features_t features) { … } static netdev_features_t nfp_net_fix_features(struct net_device *netdev, netdev_features_t features) { … } static netdev_features_t nfp_net_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { … } static int nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len) { … } static int nfp_net_xdp_setup_drv(struct nfp_net *nn, struct netdev_bpf *bpf) { … } static int nfp_net_xdp_setup_hw(struct nfp_net *nn, struct netdev_bpf *bpf) { … } static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp) { … } static int nfp_net_set_mac_address(struct net_device *netdev, void *addr) { … } static int nfp_net_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, struct net_device *dev, u32 filter_mask, int nlflags) { … } static int nfp_net_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, u16 flags, struct netlink_ext_ack *extack) { … } const struct net_device_ops nfp_nfd3_netdev_ops = …; const struct net_device_ops nfp_nfdk_netdev_ops = …; static int nfp_udp_tunnel_sync(struct net_device *netdev, unsigned int table) { … } static const struct udp_tunnel_nic_info nfp_udp_tunnels = …; /** * nfp_net_info() - Print general info about the NIC * @nn: NFP Net device to reconfigure */ void nfp_net_info(struct nfp_net *nn) { … } /** * nfp_net_alloc() - Allocate netdev and related structure * @pdev: PCI device * @dev_info: NFP ASIC params * @ctrl_bar: PCI IOMEM with vNIC config memory * @needs_netdev: Whether to allocate a netdev for this vNIC * @max_tx_rings: Maximum number of TX rings supported by device * @max_rx_rings: Maximum number of RX rings supported by device * * This function allocates a netdev device and fills in the initial * part of the @struct nfp_net structure. In case of control device * nfp_net structure is allocated without the netdev. * * Return: NFP Net device structure, or ERR_PTR on error. */ struct nfp_net * nfp_net_alloc(struct pci_dev *pdev, const struct nfp_dev_info *dev_info, void __iomem *ctrl_bar, bool needs_netdev, unsigned int max_tx_rings, unsigned int max_rx_rings) { … } /** * nfp_net_free() - Undo what @nfp_net_alloc() did * @nn: NFP Net device to reconfigure */ void nfp_net_free(struct nfp_net *nn) { … } /** * nfp_net_rss_key_sz() - Get current size of the RSS key * @nn: NFP Net device instance * * Return: size of the RSS key for currently selected hash function. */ unsigned int nfp_net_rss_key_sz(struct nfp_net *nn) { … } /** * nfp_net_rss_init() - Set the initial RSS parameters * @nn: NFP Net device to reconfigure */ static void nfp_net_rss_init(struct nfp_net *nn) { … } /** * nfp_net_irqmod_init() - Set the initial IRQ moderation parameters * @nn: NFP Net device to reconfigure */ static void nfp_net_irqmod_init(struct nfp_net *nn) { … } static void nfp_net_netdev_init(struct nfp_net *nn) { … } static int nfp_net_read_caps(struct nfp_net *nn) { … } /** * nfp_net_init() - Initialise/finalise the nfp_net structure * @nn: NFP Net device structure * * Return: 0 on success or negative errno on error. */ int nfp_net_init(struct nfp_net *nn) { … } /** * nfp_net_clean() - Undo what nfp_net_init() did. * @nn: NFP Net device structure */ void nfp_net_clean(struct nfp_net *nn) { … }
Codebase Browser
linux