// SPDX-License-Identifier: GPL-2.0 /* * PCI Peer 2 Peer DMA support. * * Copyright (c) 2016-2018, Logan Gunthorpe * Copyright (c) 2016-2017, Microsemi Corporation * Copyright (c) 2017, Christoph Hellwig * Copyright (c) 2018, Eideticom Inc. */ #define pr_fmt(fmt) … #include <linux/ctype.h> #include <linux/dma-map-ops.h> #include <linux/pci-p2pdma.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/genalloc.h> #include <linux/memremap.h> #include <linux/percpu-refcount.h> #include <linux/random.h> #include <linux/seq_buf.h> #include <linux/xarray.h> struct pci_p2pdma { … }; struct pci_p2pdma_pagemap { … }; static struct pci_p2pdma_pagemap *to_p2p_pgmap(struct dev_pagemap *pgmap) { … } static ssize_t size_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(size); static ssize_t available_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(available); static ssize_t published_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(published); static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, struct vm_area_struct *vma) { … } static struct bin_attribute p2pmem_alloc_attr = …; static struct attribute *p2pmem_attrs[] = …; static struct bin_attribute *p2pmem_bin_attrs[] = …; static const struct attribute_group p2pmem_group = …; static void p2pdma_page_free(struct page *page) { … } static const struct dev_pagemap_ops p2pdma_pgmap_ops = …; static void pci_p2pdma_release(void *data) { … } static int pci_p2pdma_setup(struct pci_dev *pdev) { … } static void pci_p2pdma_unmap_mappings(void *data) { … } /** * pci_p2pdma_add_resource - add memory for use as p2p memory * @pdev: the device to add the memory to * @bar: PCI BAR to add * @size: size of the memory to add, may be zero to use the whole BAR * @offset: offset into the PCI BAR * * The memory will be given ZONE_DEVICE struct pages so that it may * be used with any DMA request. */ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, u64 offset) { … } EXPORT_SYMBOL_GPL(…); /* * Note this function returns the parent PCI device with a * reference taken. It is the caller's responsibility to drop * the reference. */ static struct pci_dev *find_parent_pci_dev(struct device *dev) { … } /* * Check if a PCI bridge has its ACS redirection bits set to redirect P2P * TLPs upstream via ACS. Returns 1 if the packets will be redirected * upstream, 0 otherwise. */ static int pci_bridge_has_acs_redir(struct pci_dev *pdev) { … } static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev) { … } static bool cpu_supports_p2pdma(void) { … } static const struct pci_p2pdma_whitelist_entry { … } pci_p2pdma_whitelist[] = …; /* * If the first device on host's root bus is either devfn 00.0 or a PCIe * Root Port, return it. Otherwise return NULL. * * We often use a devfn 00.0 "host bridge" in the pci_p2pdma_whitelist[] * (though there is no PCI/PCIe requirement for such a device). On some * platforms, e.g., Intel Skylake, there is no such host bridge device, and * pci_p2pdma_whitelist[] may contain a Root Port at any devfn. * * This function is similar to pci_get_slot(host->bus, 0), but it does * not take the pci_bus_sem lock since __host_bridge_whitelist() must not * sleep. * * For this to be safe, the caller should hold a reference to a device on the * bridge, which should ensure the host_bridge device will not be freed * or removed from the head of the devices list. */ static struct pci_dev *pci_host_bridge_dev(struct pci_host_bridge *host) { … } static bool __host_bridge_whitelist(struct pci_host_bridge *host, bool same_host_bridge, bool warn) { … } /* * If we can't find a common upstream bridge take a look at the root * complex and compare it to a whitelist of known good hardware. */ static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b, bool warn) { … } static unsigned long map_types_idx(struct pci_dev *client) { … } /* * Calculate the P2PDMA mapping type and distance between two PCI devices. * * If the two devices are the same PCI function, return * PCI_P2PDMA_MAP_BUS_ADDR and a distance of 0. * * If they are two functions of the same device, return * PCI_P2PDMA_MAP_BUS_ADDR and a distance of 2 (one hop up to the bridge, * then one hop back down to another function of the same device). * * In the case where two devices are connected to the same PCIe switch, * return a distance of 4. This corresponds to the following PCI tree: * * -+ Root Port * \+ Switch Upstream Port * +-+ Switch Downstream Port 0 * + \- Device A * \-+ Switch Downstream Port 1 * \- Device B * * The distance is 4 because we traverse from Device A to Downstream Port 0 * to the common Switch Upstream Port, back down to Downstream Port 1 and * then to Device B. The mapping type returned depends on the ACS * redirection setting of the ports along the path. * * If ACS redirect is set on any port in the path, traffic between the * devices will go through the host bridge, so return * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; otherwise return * PCI_P2PDMA_MAP_BUS_ADDR. * * Any two devices that have a data path that goes through the host bridge * will consult a whitelist. If the host bridge is in the whitelist, return * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE with the distance set to the number of * ports per above. If the device is not in the whitelist, return * PCI_P2PDMA_MAP_NOT_SUPPORTED. */ static enum pci_p2pdma_map_type calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client, int *dist, bool verbose) { … } /** * pci_p2pdma_distance_many - Determine the cumulative distance between * a p2pdma provider and the clients in use. * @provider: p2pdma provider to check against the client list * @clients: array of devices to check (NULL-terminated) * @num_clients: number of clients in the array * @verbose: if true, print warnings for devices when we return -1 * * Returns -1 if any of the clients are not compatible, otherwise returns a * positive number where a lower number is the preferable choice. (If there's * one client that's the same as the provider it will return 0, which is best * choice). * * "compatible" means the provider and the clients are either all behind * the same PCI root port or the host bridges connected to each of the devices * are listed in the 'pci_p2pdma_whitelist'. */ int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose) { … } EXPORT_SYMBOL_GPL(…); /** * pci_has_p2pmem - check if a given PCI device has published any p2pmem * @pdev: PCI device to check */ bool pci_has_p2pmem(struct pci_dev *pdev) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pmem_find_many - find a peer-to-peer DMA memory device compatible with * the specified list of clients and shortest distance * @clients: array of devices to check (NULL-terminated) * @num_clients: number of client devices in the list * * If multiple devices are behind the same switch, the one "closest" to the * client devices in use will be chosen first. (So if one of the providers is * the same as one of the clients, that provider will be used ahead of any * other providers that are unrelated). If multiple providers are an equal * distance away, one will be chosen at random. * * Returns a pointer to the PCI device with a reference taken (use pci_dev_put * to return the reference) or NULL if no compatible device is found. The * found provider will also be assigned to the client list. */ struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients) { … } EXPORT_SYMBOL_GPL(…); /** * pci_alloc_p2pmem - allocate peer-to-peer DMA memory * @pdev: the device to allocate memory from * @size: number of bytes to allocate * * Returns the allocated memory or NULL on error. */ void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size) { … } EXPORT_SYMBOL_GPL(…); /** * pci_free_p2pmem - free peer-to-peer DMA memory * @pdev: the device the memory was allocated from * @addr: address of the memory that was allocated * @size: number of bytes that were allocated */ void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pmem_virt_to_bus - return the PCI bus address for a given virtual * address obtained with pci_alloc_p2pmem() * @pdev: the device the memory was allocated from * @addr: address of the memory that was allocated */ pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist * @pdev: the device to allocate memory from * @nents: the number of SG entries in the list * @length: number of bytes to allocate * * Return: %NULL on error or &struct scatterlist pointer and @nents on success */ struct scatterlist *pci_p2pmem_alloc_sgl(struct pci_dev *pdev, unsigned int *nents, u32 length) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl() * @pdev: the device to allocate memory from * @sgl: the allocated scatterlist */ void pci_p2pmem_free_sgl(struct pci_dev *pdev, struct scatterlist *sgl) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by * other devices with pci_p2pmem_find() * @pdev: the device with peer-to-peer DMA memory to publish * @publish: set to true to publish the memory, false to unpublish it * * Published memory can be used by other PCI device drivers for * peer-2-peer DMA operations. Non-published memory is reserved for * exclusive use of the device driver that registers the peer-to-peer * memory. */ void pci_p2pmem_publish(struct pci_dev *pdev, bool publish) { … } EXPORT_SYMBOL_GPL(…); static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct dev_pagemap *pgmap, struct device *dev) { … } /** * pci_p2pdma_map_segment - map an sg segment determining the mapping type * @state: State structure that should be declared outside of the for_each_sg() * loop and initialized to zero. * @dev: DMA device that's doing the mapping operation * @sg: scatterlist segment to map * * This is a helper to be used by non-IOMMU dma_map_sg() implementations where * the sg segment is the same for the page_link and the dma_address. * * Attempt to map a single segment in an SGL with the PCI bus address. * The segment must point to a PCI P2PDMA page and thus must be * wrapped in a is_pci_p2pdma_page(sg_page(sg)) check. * * Returns the type of mapping used and maps the page if the type is * PCI_P2PDMA_MAP_BUS_ADDR. */ enum pci_p2pdma_map_type pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev, struct scatterlist *sg) { … } /** * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store * to enable p2pdma * @page: contents of the value to be stored * @p2p_dev: returns the PCI device that was selected to be used * (if one was specified in the stored value) * @use_p2pdma: returns whether to enable p2pdma or not * * Parses an attribute value to decide whether to enable p2pdma. * The value can select a PCI device (using its full BDF device * name) or a boolean (in any format kstrtobool() accepts). A false * value disables p2pdma, a true value expects the caller * to automatically find a compatible device and specifying a PCI device * expects the caller to use the specific provider. * * pci_p2pdma_enable_show() should be used as the show operation for * the attribute. * * Returns 0 on success */ int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev, bool *use_p2pdma) { … } EXPORT_SYMBOL_GPL(…); /** * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating * whether p2pdma is enabled * @page: contents of the stored value * @p2p_dev: the selected p2p device (NULL if no device is selected) * @use_p2pdma: whether p2pdma has been enabled * * Attributes that use pci_p2pdma_enable_store() should use this function * to show the value of the attribute. * * Returns 0 on success */ ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev, bool use_p2pdma) { … } EXPORT_SYMBOL_GPL(…);
Codebase Browser
linux