// SPDX-License-Identifier: GPL-2.0 /* * Endpoint Function Driver to implement Non-Transparent Bridge functionality * Between PCI RC and EP * * Copyright (C) 2020 Texas Instruments * Copyright (C) 2022 NXP * * Based on pci-epf-ntb.c * Author: Frank Li <[email protected]> * Author: Kishon Vijay Abraham I <[email protected]> */ /* * +------------+ +---------------------------------------+ * | | | | * +------------+ | +--------------+ * | NTB | | | NTB | * | NetDev | | | NetDev | * +------------+ | +--------------+ * | NTB | | | NTB | * | Transfer | | | Transfer | * +------------+ | +--------------+ * | | | | | * | PCI NTB | | | | * | EPF | | | | * | Driver | | | PCI Virtual | * | | +---------------+ | NTB Driver | * | | | PCI EP NTB |<------>| | * | | | FN Driver | | | * +------------+ +---------------+ +--------------+ * | | | | | | * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI | * | | PCI | | | Bus | * +------------+ +---------------+--------+--------------+ * PCIe Root Port PCI EP */ #include <linux/delay.h> #include <linux/io.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/pci-epc.h> #include <linux/pci-epf.h> #include <linux/ntb.h> static struct workqueue_struct *kpcintb_workqueue; #define COMMAND_CONFIGURE_DOORBELL … #define COMMAND_TEARDOWN_DOORBELL … #define COMMAND_CONFIGURE_MW … #define COMMAND_TEARDOWN_MW … #define COMMAND_LINK_UP … #define COMMAND_LINK_DOWN … #define COMMAND_STATUS_OK … #define COMMAND_STATUS_ERROR … #define LINK_STATUS_UP … #define SPAD_COUNT … #define DB_COUNT … #define NTB_MW_OFFSET … #define DB_COUNT_MASK … #define MSIX_ENABLE … #define MAX_DB_COUNT … #define MAX_MW … enum epf_ntb_bar { … }; /* * +--------------------------------------------------+ Base * | | * | | * | | * | Common Control Register | * | | * | | * | | * +-----------------------+--------------------------+ Base+spad_offset * | | | * | Peer Spad Space | Spad Space | * | | | * | | | * +-----------------------+--------------------------+ Base+spad_offset * | | | +spad_count * 4 * | | | * | Spad Space | Peer Spad Space | * | | | * +-----------------------+--------------------------+ * Virtual PCI PCIe Endpoint * NTB Driver NTB Driver */ struct epf_ntb_ctrl { … } __packed; struct epf_ntb { … }; #define to_epf_ntb(epf_group) … #define ntb_ndev(__ntb) … static struct pci_epf_header epf_ntb_header = …; /** * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST) * @ntb: NTB device that facilitates communication between HOST and VHOST * @link_up: true or false indicating Link is UP or Down * * Once NTB function in HOST invoke ntb_link_enable(), * this NTB function driver will trigger a link event to VHOST. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) { … } /** * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST * to access the memory window of HOST * @ntb: NTB device that facilitates communication between HOST and VHOST * @mw: Index of the memory window (either 0, 1, 2 or 3) * * EP Outbound Window * +--------+ +-----------+ * | | | | * | | | | * | | | | * | | | | * | | +-----------+ * | Virtual| | Memory Win| * | NTB | -----------> | | * | Driver | | | * | | +-----------+ * | | | | * | | | | * +--------+ +-----------+ * VHOST PCI EP * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw) { … } /** * epf_ntb_teardown_mw() - Teardown the configured OB ATU * @ntb: NTB device that facilitates communication between HOST and VHOST * @mw: Index of the memory window (either 0, 1, 2 or 3) * * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using * pci_epc_unmap_addr() */ static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw) { … } /** * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST * @work: work_struct for the epf_ntb_epc * * Workqueue function that gets invoked for the two epf_ntb_epc * periodically (once every 5ms) to see if it has received any commands * from NTB HOST. The HOST can send commands to configure doorbell or * configure memory window or to update link status. */ static void epf_ntb_cmd_handler(struct work_struct *work) { … } /** * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR * @ntb: EPC associated with one of the HOST which holds peer's outbound * address. * * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and * self scratchpad region (removes inbound ATU configuration). While BAR0 is * the default self scratchpad BAR, an NTB could have other BARs for self * scratchpad (because of reserved BARs). This function can get the exact BAR * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. * * Please note the self scratchpad region and config region is combined to * a single region and mapped using the same BAR. Also note VHOST's peer * scratchpad is HOST's self scratchpad. * * Returns: void */ static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb) { … } /** * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR * @ntb: NTB device that facilitates communication between HOST and VHOST * * Map BAR0 of EP CONTROLLER which contains the VHOST's config and * self scratchpad region. * * Please note the self scratchpad region and config region is combined to * a single region and mapped using the same BAR. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb) { … } /** * epf_ntb_config_spad_bar_free() - Free the physical memory associated with * config + scratchpad region * @ntb: NTB device that facilitates communication between HOST and VHOST */ static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) { … } /** * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad * region * @ntb: NTB device that facilitates communication between HOST and VHOST * * Allocate the Local Memory mentioned in the above diagram. The size of * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION * is obtained from "spad-count" configfs entry. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb) { … } /** * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability * @ntb: NTB device that facilitates communication between HOST and VHOST * * Configure MSI/MSI-X capability for each interface with number of * interrupts equal to "db_count" configfs entry. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) { … } /** * epf_ntb_db_bar_init() - Configure Doorbell window BARs * @ntb: NTB device that facilitates communication between HOST and VHOST * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) { … } static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws); /** * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory * allocated in peer's outbound address space * @ntb: NTB device that facilitates communication between HOST and VHOST */ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb) { … } /** * epf_ntb_mw_bar_init() - Configure Memory window BARs * @ntb: NTB device that facilitates communication between HOST and VHOST * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb) { … } /** * epf_ntb_mw_bar_clear() - Clear Memory window BARs * @ntb: NTB device that facilitates communication between HOST and VHOST * @num_mws: the number of Memory window BARs that to be cleared */ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws) { … } /** * epf_ntb_epc_destroy() - Cleanup NTB EPC interface * @ntb: NTB device that facilitates communication between HOST and VHOST * * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces */ static void epf_ntb_epc_destroy(struct epf_ntb *ntb) { … } /** * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB * constructs (scratchpad region, doorbell, memorywindow) * @ntb: NTB device that facilitates communication between HOST and VHOST * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) { … } /** * epf_ntb_epc_init() - Initialize NTB interface * @ntb: NTB device that facilitates communication between HOST and VHOST * * Wrapper to initialize a particular EPC interface and start the workqueue * to check for commands from HOST. This function will write to the * EP controller HW for configuring it. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_epc_init(struct epf_ntb *ntb) { … } /** * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces * @ntb: NTB device that facilitates communication between HOST and VHOST * * Wrapper to cleanup all NTB interfaces. */ static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) { … } #define EPF_NTB_R(_name) … #define EPF_NTB_W(_name) … #define EPF_NTB_MW_R(_name) … #define EPF_NTB_MW_W(_name) … static ssize_t epf_ntb_num_mws_store(struct config_item *item, const char *page, size_t len) { … } EPF_NTB_R(…) … EPF_NTB_W(…) … EPF_NTB_R(…) … EPF_NTB_W(…) … EPF_NTB_R(…) … EPF_NTB_R(…) … EPF_NTB_W(…) … EPF_NTB_R(…) … EPF_NTB_W(…) … EPF_NTB_R(…) … EPF_NTB_W(…) … EPF_NTB_MW_R(mw1) EPF_NTB_MW_W(mw1) EPF_NTB_MW_R(mw2) EPF_NTB_MW_W(mw2) EPF_NTB_MW_R(mw3) EPF_NTB_MW_W(mw3) EPF_NTB_MW_R(mw4) EPF_NTB_MW_W(mw4) CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); CONFIGFS_ATTR(…); static struct configfs_attribute *epf_ntb_attrs[] = …; static const struct config_item_type ntb_group_type = …; /** * epf_ntb_add_cfs() - Add configfs directory specific to NTB * @epf: NTB endpoint function device * @group: A pointer to the config_group structure referencing a group of * config_items of a specific type that belong to a specific sub-system. * * Add configfs directory specific to NTB. This directory will hold * NTB specific properties like db_count, spad_count, num_mws etc., * * Returns: Pointer to config_group */ static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, struct config_group *group) { … } /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/ static u32 pci_space[] = …; static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { … } static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { … } static struct pci_ops vpci_ops = …; static int vpci_scan_bus(void *sysdata) { … } /*==================== Virtual PCIe NTB driver ==========================*/ static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx) { … } static int vntb_epf_spad_count(struct ntb_dev *ntb) { … } static int vntb_epf_peer_mw_count(struct ntb_dev *ntb) { … } static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb) { … } static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) { … } static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx, dma_addr_t addr, resource_size_t size) { … } static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) { … } static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx, phys_addr_t *base, resource_size_t *size) { … } static int vntb_epf_link_enable(struct ntb_dev *ntb, enum ntb_speed max_speed, enum ntb_width max_width) { … } static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx) { … } static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val) { … } static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx) { … } static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val) { … } static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) { … } static u64 vntb_epf_db_read(struct ntb_dev *ndev) { … } static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx, resource_size_t *addr_align, resource_size_t *size_align, resource_size_t *size_max) { … } static u64 vntb_epf_link_is_up(struct ntb_dev *ndev, enum ntb_speed *speed, enum ntb_width *width) { … } static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits) { … } static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits) { … } static int vntb_epf_link_disable(struct ntb_dev *ntb) { … } static const struct ntb_dev_ops vntb_epf_ops = …; static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id) { … } static struct pci_device_id pci_vntb_table[] = …; static struct pci_driver vntb_pci_driver = …; /* ============ PCIe EPF Driver Bind ====================*/ /** * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality * @epf: NTB endpoint function device * * Initialize both the endpoint controllers associated with NTB function device. * Invoked when a primary interface or secondary interface is bound to EPC * device. This function will succeed only when EPC is bound to both the * interfaces. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_bind(struct pci_epf *epf) { … } /** * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() * @epf: NTB endpoint function device * * Cleanup the initialization from epf_ntb_bind() */ static void epf_ntb_unbind(struct pci_epf *epf) { … } // EPF driver probe static const struct pci_epf_ops epf_ntb_ops = …; /** * epf_ntb_probe() - Probe NTB function driver * @epf: NTB endpoint function device * @id: NTB endpoint function device ID * * Probe NTB function driver when endpoint function bus detects a NTB * endpoint function. * * Returns: Zero for success, or an error code in case of failure */ static int epf_ntb_probe(struct pci_epf *epf, const struct pci_epf_device_id *id) { … } static const struct pci_epf_device_id epf_ntb_ids[] = …; static struct pci_epf_driver epf_ntb_driver = …; static int __init epf_ntb_init(void) { … } module_init(…) …; static void __exit epf_ntb_exit(void) { … } module_exit(epf_ntb_exit); MODULE_DESCRIPTION(…) …; MODULE_AUTHOR(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux