// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2020 Mellanox Technologies Ltd. */ #include <linux/module.h> #include <linux/vdpa.h> #include <linux/vringh.h> #include <uapi/linux/virtio_net.h> #include <uapi/linux/virtio_ids.h> #include <uapi/linux/vdpa.h> #include <uapi/linux/vhost_types.h> #include <linux/virtio_config.h> #include <linux/auxiliary_bus.h> #include <linux/mlx5/cq.h> #include <linux/mlx5/qp.h> #include <linux/mlx5/device.h> #include <linux/mlx5/driver.h> #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> #include <linux/mlx5/mlx5_ifc_vdpa.h> #include <linux/mlx5/mpfs.h> #include "mlx5_vdpa.h" #include "mlx5_vnet.h" MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; #define VALID_FEATURES_MASK … #define VALID_STATUS_MASK … #define MLX5_FEATURE(_mvdev, _feature) … #define MLX5V_UNTAGGED … /* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section * 5.1.6.5.5 "Device operation in multiqueue mode": * * Multiqueue is disabled by default. * The driver enables multiqueue by sending a command using class * VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue * operation, as follows: ... */ #define MLX5V_DEFAULT_VQ_COUNT … #define MLX5V_DEFAULT_VQ_SIZE … struct mlx5_vdpa_cq_buf { … }; struct mlx5_vdpa_cq { … }; struct mlx5_vdpa_umem { … }; struct mlx5_vdpa_qp { … }; struct mlx5_vq_restore_info { … }; struct mlx5_vdpa_virtqueue { … }; static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) { … } static void free_fixed_resources(struct mlx5_vdpa_net *ndev); static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev); static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled); static void teardown_vq_resources(struct mlx5_vdpa_net *ndev); static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq); static bool mlx5_vdpa_debug; #define MLX5_LOG_VIO_FLAG(_feature) … #define MLX5_LOG_VIO_STAT(_status) … /* TODO: cross-endian support */ static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) { … } static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) { … } static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) { … } static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) { … } static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) { … } static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) { … } static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) { … } static int create_tis(struct mlx5_vdpa_net *ndev) { … } static void destroy_tis(struct mlx5_vdpa_net *ndev) { … } #define MLX5_VDPA_CQE_SIZE … #define MLX5_VDPA_LOG_CQE_SIZE … static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) { … } static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) { … } static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) { … } static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) { … } static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) { … } static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) { … } static void rx_post(struct mlx5_vdpa_qp *vqp, int n) { … } static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) { … } static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) { … } static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) { … } static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, struct mlx5_vdpa_qp *vqp) { … } static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) { … } static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) { … } static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) { … } static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) { … } static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) { … } static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) { … } static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) { … } static int read_umem_params(struct mlx5_vdpa_net *ndev) { … } static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, struct mlx5_vdpa_umem **umemp) { … } static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) { … } static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) { … } static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) { … } static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static int get_queue_type(struct mlx5_vdpa_net *ndev) { … } static bool vq_is_tx(u16 idx) { … } enum { … }; static u16 get_features(u64 features) { … } static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) { … } static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev) { … } static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool filled) { … } static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) { … } static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) { … } static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, int *outlen, u32 qpn, u32 rqpn) { … } static void free_inout(void *in, void *out) { … } /* Two QPs are used by each virtqueue. One is used by the driver and one by * firmware. The fw argument indicates whether the subjected QP is the one used * by firmware. */ static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) { … } static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } struct mlx5_virtq_attr { … }; struct mlx5_virtqueue_query_mem { … }; struct mlx5_virtqueue_modify_mem { … }; static void fill_query_virtqueue_cmd(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, struct mlx5_virtqueue_query_mem *cmd) { … } static void query_virtqueue_end(struct mlx5_vdpa_net *ndev, struct mlx5_virtqueue_query_mem *cmd, struct mlx5_virtq_attr *attr) { … } static int query_virtqueues(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs, struct mlx5_virtq_attr *attrs) { … } static bool is_resumable(struct mlx5_vdpa_net *ndev) { … } static bool is_valid_state_change(int oldstate, int newstate, bool resumable) { … } static bool modifiable_virtqueue_fields(struct mlx5_vdpa_virtqueue *mvq) { … } static void fill_modify_virtqueue_cmd(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state, struct mlx5_virtqueue_modify_mem *cmd) { … } static void modify_virtqueue_end(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) { … } static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static irqreturn_t mlx5_vdpa_int_handler(int irq, void *priv) { … } static void alloc_vector(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static void dealloc_vector(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool filled) { … } static int modify_virtqueues(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs, int state) { … } static int suspend_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs) { … } static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static int resume_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs) { … } static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static int create_rqt(struct mlx5_vdpa_net *ndev) { … } #define MLX5_MODIFY_RQT_NUM_RQS … static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) { … } static void destroy_rqt(struct mlx5_vdpa_net *ndev) { … } static int create_tir(struct mlx5_vdpa_net *ndev) { … } static void destroy_tir(struct mlx5_vdpa_net *ndev) { … } #define MAX_STEERING_ENT … #define MAX_STEERING_GROUPS … #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) #define NUM_DESTS … #else #define NUM_DESTS … #endif static int add_steering_counters(struct mlx5_vdpa_net *ndev, struct macvlan_node *node, struct mlx5_flow_act *flow_act, struct mlx5_flow_destination *dests) { … } static void remove_steering_counters(struct mlx5_vdpa_net *ndev, struct macvlan_node *node) { … } static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, struct macvlan_node *node) { … } static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, struct macvlan_node *node) { … } static u64 search_val(u8 *mac, u16 vlan, bool tagged) { … } static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) { … } static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vid, bool tagged) { … } static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) { … } static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) { … } static int setup_steering(struct mlx5_vdpa_net *ndev) { … } static void teardown_steering(struct mlx5_vdpa_net *ndev) { … } static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) { … } static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) { … } static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) { … } static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) { … } static void mlx5_cvq_kick_handler(struct work_struct *work) { … } static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) { … } static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, u64 driver_area, u64 device_area) { … } static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) { … } static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) { … } static void mlx5_cvq_notify(struct vringh *vring) { … } static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) { … } static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) { … } static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) { … } static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) { … } static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) { … } static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) { … } static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) { … } static u32 mlx5_vdpa_get_vq_desc_group(struct vdpa_device *vdev, u16 idx) { … } static u64 mlx_to_vritio_features(u16 dev_features) { … } static u64 get_supported_features(struct mlx5_core_dev *mdev) { … } static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) { … } static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) { … } static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev, bool filled) { … } static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) { … } static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) { … } static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) { … } static bool get_link_state(struct mlx5_vdpa_dev *mvdev) { … } static void update_carrier(struct work_struct *work) { … } static int queue_link_work(struct mlx5_vdpa_net *ndev) { … } static int event_handler(struct notifier_block *nb, unsigned long event, void *param) { … } static void register_link_notifier(struct mlx5_vdpa_net *ndev) { … } static void unregister_link_notifier(struct mlx5_vdpa_net *ndev) { … } static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa) { … } static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) { … } static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) { … } #define MLX5_VDPA_MAX_VQ_ENTRIES … static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) { … } static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) { … } static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) { … } static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) { … } static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) { … } static int save_channels_info(struct mlx5_vdpa_net *ndev) { … } static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) { … } static void restore_channels_info(struct mlx5_vdpa_net *ndev) { … } static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *new_mr, unsigned int asid) { … } /* reslock must be held for this function */ static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled) { … } /* reslock must be held for this function */ static void teardown_vq_resources(struct mlx5_vdpa_net *ndev) { … } static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) { … } static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) { … } static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) { … } static bool needs_vqs_reset(const struct mlx5_vdpa_dev *mvdev) { … } static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags) { … } static int mlx5_vdpa_reset(struct vdpa_device *vdev) { … } static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) { … } static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, unsigned int len) { … } static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, unsigned int len) { … } static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) { … } static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, unsigned int asid) { … } static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, struct vhost_iotlb *iotlb) { … } static int mlx5_vdpa_reset_map(struct vdpa_device *vdev, unsigned int asid) { … } static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx) { … } static void free_irqs(struct mlx5_vdpa_net *ndev) { … } static void mlx5_vdpa_free(struct vdpa_device *vdev) { … } static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) { … } static int mlx5_get_vq_irq(struct vdpa_device *vdev, u16 idx) { … } static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) { … } static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, u64 *received_desc, u64 *completed_desc) { … } static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, struct sk_buff *msg, struct netlink_ext_ack *extack) { … } static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) { … } static int mlx5_vdpa_suspend(struct vdpa_device *vdev) { … } static int mlx5_vdpa_resume(struct vdpa_device *vdev) { … } static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, unsigned int asid) { … } static const struct vdpa_config_ops mlx5_vdpa_ops = …; static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) { … } static int alloc_fixed_resources(struct mlx5_vdpa_net *ndev) { … } static void free_fixed_resources(struct mlx5_vdpa_net *ndev) { … } static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev) { … } struct mlx5_vdpa_mgmtdev { … }; static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) { … } static void allocate_irqs(struct mlx5_vdpa_net *ndev) { … } static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, const struct vdpa_dev_set_config *add_config) { … } static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) { … } static int mlx5_vdpa_set_attr(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev, const struct vdpa_dev_set_config *add_config) { … } static const struct vdpa_mgmtdev_ops mdev_ops = …; static struct virtio_device_id id_table[] = …; static int mlx5v_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { … } static void mlx5v_remove(struct auxiliary_device *adev) { … } static const struct auxiliary_device_id mlx5v_id_table[] = …; MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); static struct auxiliary_driver mlx5v_driver = …; module_auxiliary_driver(…) …;
Codebase Browser
linux