linux/drivers/cxl/core/region.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/memregion.h>
#include <linux/genalloc.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/memory.h>
#include <linux/slab.h>
#include <linux/uuid.h>
#include <linux/sort.h>
#include <linux/idr.h>
#include <linux/memory-tiers.h>
#include <cxlmem.h>
#include <cxl.h>
#include "core.h"

/**
 * DOC: cxl core region
 *
 * CXL Regions represent mapped memory capacity in system physical address
 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
 * Memory ranges, Regions represent the active mapped capacity by the HDM
 * Decoder Capability structures throughout the Host Bridges, Switches, and
 * Endpoints in the topology.
 *
 * Region configuration has ordering constraints. UUID may be set at any time
 * but is only visible for persistent regions.
 * 1. Interleave granularity
 * 2. Interleave size
 * 3. Decoder targets
 */

static struct cxl_region *to_cxl_region(struct device *dev);

#define __ACCESS_ATTR_RO(_level, _name) …

#define ACCESS_DEVICE_ATTR_RO(level, name) …

#define ACCESS_ATTR_RO(level, attrib) …

ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);

#define ACCESS_ATTR_DECLARE(level, attrib) …

static struct attribute *access0_coordinate_attrs[] = …;

ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);
ACCESS_ATTR_RO(…);

static struct attribute *access1_coordinate_attrs[] = …;

#define ACCESS_VISIBLE(level) …

ACCESS_VISIBLE(0);
ACCESS_VISIBLE(1);

static const struct attribute_group cxl_region_access0_coordinate_group = …;

static const struct attribute_group *get_cxl_region_access0_group(void)
{ … }

static const struct attribute_group cxl_region_access1_coordinate_group = …;

static const struct attribute_group *get_cxl_region_access1_group(void)
{ … }

static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
			 char *buf)
{ … }

static int is_dup(struct device *match, void *data)
{ … }

static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
			  const char *buf, size_t len)
{ … }
static DEVICE_ATTR_RW(uuid);

static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
					  struct cxl_region *cxlr)
{ … }

static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
{ … }

static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
{ … }

static int commit_decoder(struct cxl_decoder *cxld)
{ … }

static int cxl_region_decode_commit(struct cxl_region *cxlr)
{ … }

static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
			    const char *buf, size_t len)
{ … }

static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
			   char *buf)
{ … }
static DEVICE_ATTR_RW(commit);

static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
				  int n)
{ … }

static ssize_t interleave_ways_show(struct device *dev,
				    struct device_attribute *attr, char *buf)
{ … }

static const struct attribute_group *get_cxl_region_target_group(void);

static ssize_t interleave_ways_store(struct device *dev,
				     struct device_attribute *attr,
				     const char *buf, size_t len)
{ … }
static DEVICE_ATTR_RW(interleave_ways);

static ssize_t interleave_granularity_show(struct device *dev,
					   struct device_attribute *attr,
					   char *buf)
{ … }

static ssize_t interleave_granularity_store(struct device *dev,
					    struct device_attribute *attr,
					    const char *buf, size_t len)
{ … }
static DEVICE_ATTR_RW(interleave_granularity);

static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
			     char *buf)
{ … }
static DEVICE_ATTR_RO(resource);

static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
			 char *buf)
{ … }
static DEVICE_ATTR_RO(mode);

static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
{ … }

static void cxl_region_iomem_release(struct cxl_region *cxlr)
{ … }

static int free_hpa(struct cxl_region *cxlr)
{ … }

static ssize_t size_store(struct device *dev, struct device_attribute *attr,
			  const char *buf, size_t len)
{ … }

static ssize_t size_show(struct device *dev, struct device_attribute *attr,
			 char *buf)
{ … }
static DEVICE_ATTR_RW(size);

static struct attribute *cxl_region_attrs[] = …;

static const struct attribute_group cxl_region_group = …;

static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
{ … }

static int check_commit_order(struct device *dev, const void *data)
{ … }

static int match_free_decoder(struct device *dev, void *data)
{ … }

static int match_auto_decoder(struct device *dev, void *data)
{ … }

static struct cxl_decoder *
cxl_region_find_decoder(struct cxl_port *port,
			struct cxl_endpoint_decoder *cxled,
			struct cxl_region *cxlr)
{ … }

static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
			  struct cxl_decoder *cxld)
{ … }

static struct cxl_region_ref *
alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
		 struct cxl_endpoint_decoder *cxled)
{ … }

static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
{ … }

static void free_region_ref(struct cxl_region_ref *cxl_rr)
{ … }

static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
			 struct cxl_endpoint_decoder *cxled)
{ … }

static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
				struct cxl_endpoint_decoder *cxled,
				struct cxl_region_ref *cxl_rr)
{ … }

/**
 * cxl_port_attach_region() - track a region's interest in a port by endpoint
 * @port: port to add a new region reference 'struct cxl_region_ref'
 * @cxlr: region to attach to @port
 * @cxled: endpoint decoder used to create or further pin a region reference
 * @pos: interleave position of @cxled in @cxlr
 *
 * The attach event is an opportunity to validate CXL decode setup
 * constraints and record metadata needed for programming HDM decoders,
 * in particular decoder target lists.
 *
 * The steps are:
 *
 * - validate that there are no other regions with a higher HPA already
 *   associated with @port
 * - establish a region reference if one is not already present
 *
 *   - additionally allocate a decoder instance that will host @cxlr on
 *     @port
 *
 * - pin the region reference by the endpoint
 * - account for how many entries in @port's target list are needed to
 *   cover all of the added endpoints.
 */
static int cxl_port_attach_region(struct cxl_port *port,
				  struct cxl_region *cxlr,
				  struct cxl_endpoint_decoder *cxled, int pos)
{ … }

static void cxl_port_detach_region(struct cxl_port *port,
				   struct cxl_region *cxlr,
				   struct cxl_endpoint_decoder *cxled)
{ … }

static int check_last_peer(struct cxl_endpoint_decoder *cxled,
			   struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
			   int distance)
{ … }

static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
{ … }

static int cxl_port_setup_targets(struct cxl_port *port,
				  struct cxl_region *cxlr,
				  struct cxl_endpoint_decoder *cxled)
{ … }

static void cxl_port_reset_targets(struct cxl_port *port,
				   struct cxl_region *cxlr)
{ … }

static void cxl_region_teardown_targets(struct cxl_region *cxlr)
{ … }

static int cxl_region_setup_targets(struct cxl_region *cxlr)
{ … }

static int cxl_region_validate_position(struct cxl_region *cxlr,
					struct cxl_endpoint_decoder *cxled,
					int pos)
{ … }

static int cxl_region_attach_position(struct cxl_region *cxlr,
				      struct cxl_root_decoder *cxlrd,
				      struct cxl_endpoint_decoder *cxled,
				      const struct cxl_dport *dport, int pos)
{ … }

static int cxl_region_attach_auto(struct cxl_region *cxlr,
				  struct cxl_endpoint_decoder *cxled, int pos)
{ … }

static int cmp_interleave_pos(const void *a, const void *b)
{ … }

static struct cxl_port *next_port(struct cxl_port *port)
{ … }

static int match_switch_decoder_by_range(struct device *dev, void *data)
{ … }

static int find_pos_and_ways(struct cxl_port *port, struct range *range,
			     int *pos, int *ways)
{ … }

/**
 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
 * @cxled: endpoint decoder member of given region
 *
 * The endpoint position is calculated by traversing the topology from
 * the endpoint to the root decoder and iteratively applying this
 * calculation:
 *
 *    position = position * parent_ways + parent_pos;
 *
 * ...where @position is inferred from switch and root decoder target lists.
 *
 * Return: position >= 0 on success
 *	   -ENXIO on failure
 */
static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
{ … }

static int cxl_region_sort_targets(struct cxl_region *cxlr)
{ … }

static int cxl_region_attach(struct cxl_region *cxlr,
			     struct cxl_endpoint_decoder *cxled, int pos)
{ … }

static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
{ … }

void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
{ … }

static int attach_target(struct cxl_region *cxlr,
			 struct cxl_endpoint_decoder *cxled, int pos,
			 unsigned int state)
{ … }

static int detach_target(struct cxl_region *cxlr, int pos)
{ … }

static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
			    size_t len)
{ … }

#define TARGET_ATTR_RW(n) …

TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);
TARGET_ATTR_RW(…);

static struct attribute *target_attrs[] = …;

static umode_t cxl_region_target_visible(struct kobject *kobj,
					 struct attribute *a, int n)
{ … }

static const struct attribute_group cxl_region_target_group = …;

static const struct attribute_group *get_cxl_region_target_group(void)
{ … }

static const struct attribute_group *region_groups[] = …;

static void cxl_region_release(struct device *dev)
{ … }

const struct device_type cxl_region_type = …;

bool is_cxl_region(struct device *dev)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

static struct cxl_region *to_cxl_region(struct device *dev)
{ … }

static void unregister_region(void *_cxlr)
{ … }

static struct lock_class_key cxl_region_key;

static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
{ … }

static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
{ … }

static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
					  unsigned long action, void *arg)
{ … }

static int cxl_region_calculate_adistance(struct notifier_block *nb,
					  unsigned long nid, void *data)
{ … }

/**
 * devm_cxl_add_region - Adds a region to a decoder
 * @cxlrd: root decoder
 * @id: memregion id to create, or memregion_free() on failure
 * @mode: mode for the endpoint decoders of this region
 * @type: select whether this is an expander or accelerator (type-2 or type-3)
 *
 * This is the second step of region initialization. Regions exist within an
 * address space which is mapped by a @cxlrd.
 *
 * Return: 0 if the region was added to the @cxlrd, else returns negative error
 * code. The region will be named "regionZ" where Z is the unique region number.
 */
static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
					      int id,
					      enum cxl_decoder_mode mode,
					      enum cxl_decoder_type type)
{ … }

static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
{ … }

static ssize_t create_pmem_region_show(struct device *dev,
				       struct device_attribute *attr, char *buf)
{ … }

static ssize_t create_ram_region_show(struct device *dev,
				      struct device_attribute *attr, char *buf)
{ … }

static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
					  enum cxl_decoder_mode mode, int id)
{ … }

static ssize_t create_pmem_region_store(struct device *dev,
					struct device_attribute *attr,
					const char *buf, size_t len)
{ … }
DEVICE_ATTR_RW(…);

static ssize_t create_ram_region_store(struct device *dev,
				       struct device_attribute *attr,
				       const char *buf, size_t len)
{ … }
DEVICE_ATTR_RW(…);

static ssize_t region_show(struct device *dev, struct device_attribute *attr,
			   char *buf)
{ … }
DEVICE_ATTR_RO(…);

static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
{ … }

static ssize_t delete_region_store(struct device *dev,
				   struct device_attribute *attr,
				   const char *buf, size_t len)
{ … }
DEVICE_ATTR_WO(…);

static void cxl_pmem_region_release(struct device *dev)
{ … }

static const struct attribute_group *cxl_pmem_region_attribute_groups[] = …;

const struct device_type cxl_pmem_region_type = …;

bool is_cxl_pmem_region(struct device *dev)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

struct cxl_poison_context { … };

static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
				   struct cxl_poison_context *ctx)
{ … }

static int poison_by_decoder(struct device *dev, void *arg)
{ … }

int cxl_get_poison_by_endpoint(struct cxl_port *port)
{ … }

struct cxl_dpa_to_region_context { … };

static int __cxl_dpa_to_region(struct device *dev, void *arg)
{ … }

struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
{ … }

static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
{ … }

u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
		   u64 dpa)
{ … }

static struct lock_class_key cxl_pmem_region_key;

static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
{ … }

static void cxl_dax_region_release(struct device *dev)
{ … }

static const struct attribute_group *cxl_dax_region_attribute_groups[] = …;

const struct device_type cxl_dax_region_type = …;

static bool is_cxl_dax_region(struct device *dev)
{ … }

struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

static struct lock_class_key cxl_dax_region_key;

static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
{ … }

static void cxlr_pmem_unregister(void *_cxlr_pmem)
{ … }

static void cxlr_release_nvdimm(void *_cxlr)
{ … }

/**
 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
 * @cxlr: parent CXL region for this pmem region bridge device
 *
 * Return: 0 on success negative error code on failure.
 */
static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
{ … }

static void cxlr_dax_unregister(void *_cxlr_dax)
{ … }

static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
{ … }

static int match_root_decoder_by_range(struct device *dev, void *data)
{ … }

static int match_region_by_range(struct device *dev, void *data)
{ … }

/* Establish an empty region covering the given HPA range */
static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
					   struct cxl_endpoint_decoder *cxled)
{ … }

int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
{ … }
EXPORT_SYMBOL_NS_GPL(…);

static int is_system_ram(struct resource *res, void *arg)
{ … }

static void shutdown_notifiers(void *_cxlr)
{ … }

static int cxl_region_probe(struct device *dev)
{ … }

static struct cxl_driver cxl_region_driver = …;

int cxl_region_init(void)
{ … }

void cxl_region_exit(void)
{ … }

MODULE_IMPORT_NS(…);
MODULE_IMPORT_NS(…);
MODULE_ALIAS_CXL(…);