// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Copyright (c) 2021, Microsoft Corporation. */ #include <uapi/linux/bpf.h> #include <linux/inetdevice.h> #include <linux/etherdevice.h> #include <linux/ethtool.h> #include <linux/filter.h> #include <linux/mm.h> #include <linux/pci.h> #include <net/checksum.h> #include <net/ip6_checksum.h> #include <net/page_pool/helpers.h> #include <net/xdp.h> #include <net/mana/mana.h> #include <net/mana/mana_auxiliary.h> static DEFINE_IDA(mana_adev_ida); static int mana_adev_idx_alloc(void) { … } static void mana_adev_idx_free(int idx) { … } /* Microsoft Azure Network Adapter (MANA) functions */ static int mana_open(struct net_device *ndev) { … } static int mana_close(struct net_device *ndev) { … } static bool mana_can_tx(struct gdma_queue *wq) { … } static unsigned int mana_checksum_info(struct sk_buff *skb) { … } static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) { … } static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, struct mana_tx_package *tp, int gso_hs) { … } /* Handle the case when GSO SKB linear length is too large. * MANA NIC requires GSO packets to put only the packet header to SGE0. * So, we need 2 SGEs for the skb linear part which contains more than the * header. * Return a positive value for the number of SGEs, or a negative value * for an error. */ static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, int gso_hs) { … } /* Get the GSO packet's header size */ static int mana_get_gso_hs(struct sk_buff *skb) { … } netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) { … } static void mana_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *st) { … } static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, int old_q) { … } static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, struct net_device *sb_dev) { … } /* Release pre-allocated RX buffers */ static void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) { … } /* Get a buffer from the pre-allocated RX buffers */ static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) { … } /* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ static void mana_get_rxbuf_cfg(int mtu, u32 *datasize, u32 *alloc_size, u32 *headroom) { … } static int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu) { … } static int mana_change_mtu(struct net_device *ndev, int new_mtu) { … } static const struct net_device_ops mana_devops = …; static void mana_cleanup_port_context(struct mana_port_context *apc) { … } static void mana_cleanup_indir_table(struct mana_port_context *apc) { … } static int mana_init_port_context(struct mana_port_context *apc) { … } static int mana_send_request(struct mana_context *ac, void *in_buf, u32 in_len, void *out_buf, u32 out_len) { … } static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, const enum mana_command_code expected_code, const u32 min_size) { … } static int mana_pf_register_hw_vport(struct mana_port_context *apc) { … } static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) { … } static int mana_pf_register_filter(struct mana_port_context *apc) { … } static void mana_pf_deregister_filter(struct mana_port_context *apc) { … } static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, u32 proto_minor_ver, u32 proto_micro_ver, u16 *max_num_vports) { … } static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) { … } void mana_uncfg_vport(struct mana_port_context *apc) { … } EXPORT_SYMBOL_NS(…); int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, u32 doorbell_pg_id) { … } EXPORT_SYMBOL_NS(…); static int mana_cfg_vport_steering(struct mana_port_context *apc, enum TRI_STATE rx, bool update_default_rxobj, bool update_key, bool update_tab) { … } int mana_create_wq_obj(struct mana_port_context *apc, mana_handle_t vport, u32 wq_type, struct mana_obj_spec *wq_spec, struct mana_obj_spec *cq_spec, mana_handle_t *wq_obj) { … } EXPORT_SYMBOL_NS(…); void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, mana_handle_t wq_obj) { … } EXPORT_SYMBOL_NS(…); static void mana_destroy_eq(struct mana_context *ac) { … } static int mana_create_eq(struct mana_context *ac) { … } static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) { … } static void mana_fence_rqs(struct mana_port_context *apc) { … } static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) { … } static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) { … } static void mana_poll_tx_cq(struct mana_cq *cq) { … } static void mana_post_pkt_rxq(struct mana_rxq *rxq) { … } static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, uint pkt_len, struct xdp_buff *xdp) { … } static void mana_rx_skb(void *buf_va, bool from_pool, struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) { … } static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, dma_addr_t *da, bool *from_pool, bool is_napi) { … } /* Allocate frag for rx buffer, and save the old buf */ static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, struct mana_recv_buf_oob *rxoob, void **old_buf, bool *old_fp) { … } static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, struct gdma_comp *cqe) { … } static void mana_poll_rx_cq(struct mana_cq *cq) { … } static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) { … } static int mana_poll(struct napi_struct *napi, int budget) { … } static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) { … } static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) { … } static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) { … } static void mana_destroy_txq(struct mana_port_context *apc) { … } static int mana_create_txq(struct mana_port_context *apc, struct net_device *net) { … } static void mana_destroy_rxq(struct mana_port_context *apc, struct mana_rxq *rxq, bool validate_state) { … } static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, struct mana_rxq *rxq, struct device *dev) { … } #define MANA_WQE_HEADER_SIZE … #define MANA_WQE_SGE_SIZE … static int mana_alloc_rx_wqe(struct mana_port_context *apc, struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) { … } static int mana_push_wqe(struct mana_rxq *rxq) { … } static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) { … } static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, u32 rxq_idx, struct mana_eq *eq, struct net_device *ndev) { … } static int mana_add_rx_queues(struct mana_port_context *apc, struct net_device *ndev) { … } static void mana_destroy_vport(struct mana_port_context *apc) { … } static int mana_create_vport(struct mana_port_context *apc, struct net_device *net) { … } static int mana_rss_table_alloc(struct mana_port_context *apc) { … } static void mana_rss_table_init(struct mana_port_context *apc) { … } int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, bool update_hash, bool update_tab) { … } void mana_query_gf_stats(struct mana_port_context *apc) { … } static int mana_init_port(struct net_device *ndev) { … } int mana_alloc_queues(struct net_device *ndev) { … } int mana_attach(struct net_device *ndev) { … } static int mana_dealloc_queues(struct net_device *ndev) { … } int mana_detach(struct net_device *ndev, bool from_close) { … } static int mana_probe_port(struct mana_context *ac, int port_idx, struct net_device **ndev_storage) { … } static void adev_release(struct device *dev) { … } static void remove_adev(struct gdma_dev *gd) { … } static int add_adev(struct gdma_dev *gd) { … } int mana_probe(struct gdma_dev *gd, bool resuming) { … } void mana_remove(struct gdma_dev *gd, bool suspending) { … } struct net_device *mana_get_primary_netdev_rcu(struct mana_context *ac, u32 port_index) { … } EXPORT_SYMBOL_NS(…);
Codebase Browser
linux