linux/drivers/infiniband/hw/mlx5/fs.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
 */

#include <rdma/ib_user_verbs.h>
#include <rdma/ib_verbs.h>
#include <rdma/uverbs_types.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/uverbs_std_types.h>
#include <rdma/mlx5_user_ioctl_cmds.h>
#include <rdma/mlx5_user_ioctl_verbs.h>
#include <rdma/ib_hdrs.h>
#include <rdma/ib_umem.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/fs.h>
#include <linux/mlx5/fs_helpers.h>
#include <linux/mlx5/eswitch.h>
#include <net/inet_ecn.h>
#include "mlx5_ib.h"
#include "counters.h"
#include "devx.h"
#include "fs.h"

#define UVERBS_MODULE_NAME …
#include <rdma/uverbs_named_ioctl.h>

enum { … };

#define HEADER_IS_ZERO(match_criteria, headers) …       \

static u8 get_match_criteria_enable(u32 *match_criteria)
{ … }

static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
{ … }

static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
			   bool inner)
{ … }

static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
{ … }

static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
{ … }

#define LAST_ETH_FIELD …
#define LAST_IPV4_FIELD …
#define LAST_IPV6_FIELD …
#define LAST_TCP_UDP_FIELD …
#define LAST_TUNNEL_FIELD …
#define LAST_FLOW_TAG_FIELD …
#define LAST_DROP_FIELD …
#define LAST_COUNTERS_FIELD …

/* Field is the last supported field */
#define FIELDS_NOT_SUPPORTED(filter, field) …

int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
			   bool is_egress,
			   struct mlx5_flow_act *action)
{ … }

static int parse_flow_attr(struct mlx5_core_dev *mdev,
			   struct mlx5_flow_spec *spec,
			   const union ib_flow_spec *ib_spec,
			   const struct ib_flow_attr *flow_attr,
			   struct mlx5_flow_act *action, u32 prev_type)
{ … }

/* If a flow could catch both multicast and unicast packets,
 * it won't fall into the multicast flow steering table and this rule
 * could steal other multicast packets.
 */
static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
{ … }

static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
			       const struct ib_flow_attr *flow_attr,
			       bool check_inner)
{ … }

static bool is_valid_attr(struct mlx5_core_dev *mdev,
			  const struct ib_flow_attr *flow_attr)
{ … }

static void put_flow_table(struct mlx5_ib_dev *dev,
			   struct mlx5_ib_flow_prio *prio, bool ft_added)
{ … }

static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
{ … }

static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
{ … }

enum flow_table_type { … };

#define MLX5_FS_MAX_TYPES …
#define MLX5_FS_MAX_ENTRIES …

static bool mlx5_ib_shared_ft_allowed(struct ib_device *device)
{ … }

static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_ib_dev *dev,
					   struct mlx5_flow_namespace *ns,
					   struct mlx5_ib_flow_prio *prio,
					   int priority,
					   int num_entries, int num_groups,
					   u32 flags)
{ … }

static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
						struct ib_flow_attr *flow_attr,
						enum flow_table_type ft_type)
{ … }

enum { … };

enum { … };

static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
			      struct mlx5_flow_spec *spec)
{ … }

static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
			   struct mlx5_flow_spec *spec, int ipv)
{ … }

static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
			struct mlx5_flow_spec *spec)
{ … }

int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
			 struct mlx5_ib_op_fc *opfc,
			 enum mlx5_ib_optional_counter_type type)
{ … }

void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
			     struct mlx5_ib_op_fc *opfc,
			     enum mlx5_ib_optional_counter_type type)
{ … }

static void set_underlay_qp(struct mlx5_ib_dev *dev,
			    struct mlx5_flow_spec *spec,
			    u32 underlay_qpn)
{ … }

static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
					 struct mlx5_flow_spec *spec,
					 struct mlx5_eswitch_rep *rep)
{ … }

static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
						      struct mlx5_ib_flow_prio *ft_prio,
						      const struct ib_flow_attr *flow_attr,
						      struct mlx5_flow_destination *dst,
						      u32 underlay_qpn,
						      struct mlx5_ib_create_flow *ucmd)
{ … }

static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
						     struct mlx5_ib_flow_prio *ft_prio,
						     const struct ib_flow_attr *flow_attr,
						     struct mlx5_flow_destination *dst)
{ … }

enum { … };

static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
							  struct mlx5_ib_flow_prio *ft_prio,
							  struct ib_flow_attr *flow_attr,
							  struct mlx5_flow_destination *dst)
{ … }

static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
							struct mlx5_ib_flow_prio *ft_rx,
							struct mlx5_ib_flow_prio *ft_tx,
							struct mlx5_flow_destination *dst)
{ … }

static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
					   struct ib_flow_attr *flow_attr,
					   struct ib_udata *udata)
{ … }

static struct mlx5_ib_flow_prio *
_get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
		enum mlx5_flow_namespace_type ns_type,
		bool mcast)
{ … }

static struct mlx5_ib_flow_handler *
_create_raw_flow_rule(struct mlx5_ib_dev *dev,
		      struct mlx5_ib_flow_prio *ft_prio,
		      struct mlx5_flow_destination *dst,
		      struct mlx5_ib_flow_matcher  *fs_matcher,
		      struct mlx5_flow_context *flow_context,
		      struct mlx5_flow_act *flow_act,
		      void *cmd_in, int inlen,
		      int dst_num)
{ … }

static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
				void *match_v)
{ … }

static struct mlx5_ib_flow_handler *raw_fs_rule_add(
	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
	struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
	u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type)
{ … }

static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
{ … }

static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
{ … }

static int
mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
			     enum mlx5_flow_namespace_type *namespace)
{ … }

static const struct uverbs_attr_spec mlx5_ib_flow_type[] = …;

static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
{ … }

static int get_dests(struct uverbs_attr_bundle *attrs,
		     struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
		     int *dest_type, struct ib_qp **qp, u32 *flags)
{ … }

static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id)
{ … }

#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS …
static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
	struct uverbs_attr_bundle *attrs)
{ … }

static int flow_matcher_cleanup(struct ib_uobject *uobject,
				enum rdma_remove_reason why,
				struct uverbs_attr_bundle *attrs)
{ … }

static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
				     struct mlx5_ib_flow_prio *ft_prio,
				     enum mlx5_flow_namespace_type ns_type)
{ … }

static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int
steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static void
steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int
steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static void
steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int
steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int
steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static void
steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
				      struct mlx5_ib_flow_prio *ft_prio,
				      enum mlx5_flow_namespace_type ns_type)
{ … }

static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
{ … }

static int steering_anchor_cleanup(struct ib_uobject *uobject,
				   enum rdma_remove_reason why,
				   struct uverbs_attr_bundle *attrs)
{ … }

static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
			      int count)
{ … }

void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
{ … }

static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
			      struct mlx5_ib_flow_matcher *obj)
{ … }

static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
	struct uverbs_attr_bundle *attrs)
{ … }

static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
	struct uverbs_attr_bundle *attrs)
{ … }

static struct ib_flow_action *
mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
			     enum mlx5_ib_uapi_flow_table_type ft_type,
			     u8 num_actions, void *in)
{ … }

static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
{ … }

static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
	struct uverbs_attr_bundle *attrs)
{ … }

static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
						      u8 packet_reformat_type,
						      u8 ft_type)
{ … }

static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
{ … }

static int mlx5_ib_flow_action_create_packet_reformat_ctx(
	struct mlx5_ib_dev *dev,
	struct mlx5_ib_flow_action *maction,
	u8 ft_type, u8 dv_prt,
	void *in, size_t len)
{ … }

static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
	struct uverbs_attr_bundle *attrs)
{ … }

DECLARE_UVERBS_NAMED_METHOD(…);

DECLARE_UVERBS_NAMED_METHOD_DESTROY(…);

ADD_UVERBS_METHODS(mlx5_ib_fs,
		   UVERBS_OBJECT_FLOW,
		   &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
		   &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));

DECLARE_UVERBS_NAMED_METHOD(…);

DECLARE_UVERBS_NAMED_METHOD(…);

ADD_UVERBS_METHODS(
	mlx5_ib_flow_actions,
	UVERBS_OBJECT_FLOW_ACTION,
	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));

DECLARE_UVERBS_NAMED_METHOD(…);

DECLARE_UVERBS_NAMED_METHOD_DESTROY(…);

DECLARE_UVERBS_NAMED_OBJECT(…);

DECLARE_UVERBS_NAMED_METHOD(…);

DECLARE_UVERBS_NAMED_METHOD_DESTROY(…);

DECLARE_UVERBS_NAMED_OBJECT(…);

const struct uapi_definition mlx5_ib_flow_defs[] = …;

static const struct ib_device_ops flow_ops = …;

int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
{ … }