/* * Copyright 2023 Red Hat Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "priv.h" #include <core/pci.h> #include <subdev/timer.h> #include <subdev/vfn.h> #include <engine/fifo/chan.h> #include <engine/sec2.h> #include <nvfw/fw.h> #include <nvrm/nvtypes.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h> #include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h> #include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h> #include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h> #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h> #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h> #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h> #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h> #include <nvrm/535.113.01/nvidia/generated/g_allclasses.h> #include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h> #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h> #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h> #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h> #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h> #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h> #include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h> #include <linux/acpi.h> #include <linux/ctype.h> #include <linux/parser.h> #define GSP_MSG_MIN_SIZE … #define GSP_MSG_MAX_SIZE … struct r535_gsp_msg { … }; #define GSP_MSG_HDR_SIZE … static int r535_rpc_status_to_errno(uint32_t rpc_status) { … } static void * r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime) { … } static void * r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime) { … } static int r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv) { … } static void * r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc) { … } struct nvfw_gsp_rpc { … }; static void r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg) { … } static void r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl) { … } static struct nvfw_gsp_rpc * r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc) { … } static int r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv) { … } static int r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn) { … } static void * r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc) { … } static void r535_gsp_event_dtor(struct nvkm_gsp_event *event) { … } static int r535_gsp_device_event_get(struct nvkm_gsp_event *event) { … } static int r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id, nvkm_gsp_event_func func, struct nvkm_gsp_event *event) { … } static void r535_gsp_device_dtor(struct nvkm_gsp_device *device) { … } static int r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device) { … } static int r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device) { … } static void r535_gsp_client_dtor(struct nvkm_gsp_client *client) { … } static int r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client) { … } static int r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object) { … } static void r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv) { … } static void * r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc) { … } static void * r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc) { … } static void r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv) { … } static int r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc) { … } static void * r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc) { … } static void r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv) { … } static void * r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc) { … } static void * r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc) { … } const struct nvkm_gsp_rm r535_gsp_rm = …; static void r535_gsp_msgq_work(struct work_struct *work) { … } static irqreturn_t r535_gsp_intr(struct nvkm_inth *inth) { … } static int r535_gsp_intr_get_table(struct nvkm_gsp *gsp) { … } static int r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp) { … } static void nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem) { … } static int nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem) { … } static int r535_gsp_postinit(struct nvkm_gsp *gsp) { … } static int r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend) { … } enum registry_type { … }; /* An arbitrary limit to the length of a registry key */ #define REGISTRY_MAX_KEY_LENGTH … /** * registry_list_entry - linked list member for a registry key/value * @head: list_head struct * @type: dword, binary, or string * @klen: the length of name of the key * @vlen: the length of the value * @key: the key name * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD * @binary: the data, if TYPE_BINARY or TYPE_STRING * * Every registry key/value is represented internally by this struct. * * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in * @dword. * * Types BINARY and STRING are variable-length binary blobs. The only real * difference between BINARY and STRING is that STRING is null-terminated and * is expected to contain only printable characters. * * Note: it is technically possible to have multiple keys with the same name * but different types, but this is not useful since GSP-RM expects keys to * have only one specific type. */ struct registry_list_entry { … }; /** * add_registry -- adds a registry entry * @gsp: gsp pointer * @key: name of the registry key * @type: type of data * @data: pointer to value * @length: size of data, in bytes * * Adds a registry key/value pair to the registry database. * * This function collects the registry information in a linked list. After * all registry keys have been added, build_registry() is used to create the * RPC data structure. * * registry_rpc_size is a running total of the size of all registry keys. * It's used to avoid an O(n) calculation of the size when the RPC is built. * * Returns 0 on success, or negative error code on error. */ static int add_registry(struct nvkm_gsp *gsp, const char *key, enum registry_type type, const void *data, size_t length) { … } static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value) { … } static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value) { … } /** * build_registry -- create the registry RPC data * @gsp: gsp pointer * @registry: pointer to the RPC payload to fill * * After all registry key/value pairs have been added, call this function to * build the RPC. * * The registry RPC looks like this: * * +-----------------+ * |NvU32 size; | * |NvU32 numEntries;| * +-----------------+ * +----------------------------------------+ * |PACKED_REGISTRY_ENTRY | * +----------------------------------------+ * |Null-terminated key (string) for entry 0| * +----------------------------------------+ * |Binary/string data value for entry 0 | (only if necessary) * +----------------------------------------+ * * +----------------------------------------+ * |PACKED_REGISTRY_ENTRY | * +----------------------------------------+ * |Null-terminated key (string) for entry 1| * +----------------------------------------+ * |Binary/string data value for entry 1 | (only if necessary) * +----------------------------------------+ * ... (and so on, one copy for each entry) * * * The 'data' field of an entry is either a 32-bit integer (for type DWORD) * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING). * * All memory allocated by add_registry() is released. */ static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry) { … } /** * clean_registry -- clean up registry memory in case of error * @gsp: gsp pointer * * Call this function to clean up all memory allocated by add_registry() * in case of error and build_registry() is not called. */ static void clean_registry(struct nvkm_gsp *gsp) { … } MODULE_PARM_DESC(…) …; static char *NVreg_RegistryDwords; module_param(NVreg_RegistryDwords, charp, 0400); /* dword only */ struct nv_gsp_registry_entries { … }; /** * r535_registry_entries - required registry entries for GSP-RM * * This array lists registry entries that are required for GSP-RM to * function correctly. * * RMSecBusResetEnable - enables PCI secondary bus reset * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration * registers on any PCI reset. */ static const struct nv_gsp_registry_entries r535_registry_entries[] = …; #define NV_GSP_REG_NUM_ENTRIES … /** * strip - strips all characters in 'reject' from 's' * @s: string to strip * @reject: string of characters to remove * * 's' is modified. * * Returns the length of the new string. */ static size_t strip(char *s, const char *reject) { … } /** * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM * @gsp: gsp pointer * * The GSP-RM registry is a set of key/value pairs that configure some aspects * of GSP-RM. The keys are strings, and the values are 32-bit integers. * * The registry is built from a combination of a static hard-coded list (see * above) and entries passed on the driver's command line. */ static int r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp) { … } #if defined(CONFIG_ACPI) && defined(CONFIG_X86) static void r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps) { … } static void r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt) { … } static void r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode, MUX_METHOD_DATA_ELEMENT *part) { … } static void r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux) { … } static void r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod) { … } #endif static void r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi) { … } static int r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp) { … } static int r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc) { … } static int r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc) { … } static int r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc) { … } static int r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc) { … } /** * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP * @priv: gsp pointer * @fn: function number (ignored) * @repv: pointer to libos print RPC * @repc: message size * * The GSP sequencer is a list of I/O commands that the GSP can send to * the driver to perform for various purposes. The most common usage is to * perform a special mid-initialization reset. */ static int r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc) { … } static int r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1) { … } static int r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1) { … } static int r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp) { … } static int r535_gsp_shared_init(struct nvkm_gsp *gsp) { … } static int r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume) { … } static inline u64 r535_gsp_libos_id8(const char *name) { … } /** * create_pte_array() - creates a PTE array of a physically contiguous buffer * @ptes: pointer to the array * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned) * @size: size of the buffer * * GSP-RM sometimes expects physically-contiguous buffers to have an array of * "PTEs" for each page in that buffer. Although in theory that allows for * the buffer to be physically discontiguous, GSP-RM does not currently * support that. * * In this case, the PTEs are DMA addresses of each page of the buffer. Since * the buffer is physically contiguous, calculating all the PTEs is simple * math. * * See memdescGetPhysAddrsForGpu() */ static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size) { … } /** * r535_gsp_libos_init() -- create the libos arguments structure * @gsp: gsp pointer * * The logging buffers are byte queues that contain encoded printf-like * messages from GSP-RM. They need to be decoded by a special application * that can parse the buffers. * * The 'loginit' buffer contains logs from early GSP-RM init and * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE. * * The physical address map for the log buffer is stored in the buffer * itself, starting with offset 1. Offset 0 contains the "put" pointer. * * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is * configured for a larger page size (e.g. 64K pages), we need to give * the GSP an array of 4K pages. Fortunately, since the buffer is * physically contiguous, it's simple math to calculate the addresses. * * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the * buffers to be physically contiguous anyway. * * The memory allocated for the arguments must remain until the GSP sends the * init_done RPC. * * See _kgspInitLibosLoggingStructures (allocates memory for buffers) * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) */ static int r535_gsp_libos_init(struct nvkm_gsp *gsp) { … } void nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt) { … } int nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt) { … } static void nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3) { … } /** * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list * @gsp: gsp pointer * @sgt: S/G list to traverse * @size: size of the image, in bytes * @rx3: radix3 array to update * * The GSP uses a three-level page table, called radix3, to map the firmware. * Each 64-bit "pointer" in the table is either the bus address of an entry in * the next table (for levels 0 and 1) or the bus address of the next page in * the GSP firmware image itself. * * Level 0 contains a single entry in one page that points to the first page * of level 1. * * Level 1, since it's also only one page in size, contains up to 512 entries, * one for each page in Level 2. * * Level 2 can be up to 512 pages in size, and each of those entries points to * the next page of the firmware image. Since there can be up to 512*512 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB. * * Internally, the GSP has its window into system memory, but the base * physical address of the aperture is not 0. In fact, it varies depending on * the GPU architecture. Since the GPU is a PCI device, this window is * accessed via DMA and is therefore bound by IOMMU translation. The end * result is that GSP-RM must translate the bus addresses in the table to GSP * physical addresses. All this should happen transparently. * * Returns 0 on success, or negative error code * * See kgspCreateRadix3_IMPL */ static int nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, struct nvkm_gsp_radix3 *rx3) { … } int r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend) { … } int r535_gsp_init(struct nvkm_gsp *gsp) { … } static int r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp) { … } static const struct nvkm_firmware_func r535_gsp_fw = …; static int r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize) { … } static void r535_gsp_dtor_fws(struct nvkm_gsp *gsp) { … } void r535_gsp_dtor(struct nvkm_gsp *gsp) { … } int r535_gsp_oneinit(struct nvkm_gsp *gsp) { … } static int r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver, const struct firmware **pfw) { … } int r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif) { … } #define NVKM_GSP_FIRMWARE(chip) … NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…); NVKM_GSP_FIRMWARE(…);
Codebase Browser
linux