// SPDX-License-Identifier: GPL-2.0-or-later /* * Driver for the Micron P320 SSD * Copyright (C) 2011 Micron Technology, Inc. * * Portions of this code were derived from works subjected to the * following copyright: * Copyright (C) 2009 Integrated Device Technology, Inc. */ #include <linux/pci.h> #include <linux/interrupt.h> #include <linux/ata.h> #include <linux/delay.h> #include <linux/hdreg.h> #include <linux/uaccess.h> #include <linux/random.h> #include <linux/smp.h> #include <linux/compat.h> #include <linux/fs.h> #include <linux/module.h> #include <linux/blkdev.h> #include <linux/blk-mq.h> #include <linux/bio.h> #include <linux/dma-mapping.h> #include <linux/idr.h> #include <linux/kthread.h> #include <../drivers/ata/ahci.h> #include <linux/export.h> #include <linux/debugfs.h> #include <linux/prefetch.h> #include <linux/numa.h> #include "mtip32xx.h" #define HW_CMD_SLOT_SZ … /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */ #define AHCI_RX_FIS_SZ … #define AHCI_RX_FIS_OFFSET … #define AHCI_IDFY_SZ … #define AHCI_IDFY_OFFSET … #define AHCI_SECTBUF_SZ … #define AHCI_SECTBUF_OFFSET … #define AHCI_SMARTBUF_SZ … #define AHCI_SMARTBUF_OFFSET … /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */ #define BLOCK_DMA_ALLOC_SZ … /* DMA region containing command table (should be 8192 bytes) */ #define AHCI_CMD_SLOT_SZ … #define AHCI_CMD_TBL_SZ … #define AHCI_CMD_TBL_OFFSET … /* DMA region per command (contains header and SGL) */ #define AHCI_CMD_TBL_HDR_SZ … #define AHCI_CMD_TBL_HDR_OFFSET … #define AHCI_CMD_TBL_SGL_SZ … #define AHCI_CMD_TBL_SGL_OFFSET … #define CMD_DMA_ALLOC_SZ … #define HOST_CAP_NZDMA … #define HOST_HSORG … #define HSORG_DISABLE_SLOTGRP_INTR … #define HSORG_DISABLE_SLOTGRP_PXIS … #define HSORG_HWREV … #define HSORG_STYLE … #define HSORG_SLOTGROUPS … #define PORT_COMMAND_ISSUE … #define PORT_SDBV … #define PORT_OFFSET … #define PORT_MEM_SIZE … #define PORT_IRQ_ERR … #define PORT_IRQ_LEGACY … #define PORT_IRQ_HANDLED … #define DEF_PORT_IRQ … /* product numbers */ #define MTIP_PRODUCT_UNKNOWN … #define MTIP_PRODUCT_ASICFPGA … /* Device instance number, incremented each time a device is probed. */ static int instance; /* * Global variable used to hold the major block device number * allocated in mtip_init(). */ static int mtip_major; static struct dentry *dfs_parent; static u32 cpu_use[NR_CPUS]; static DEFINE_IDA(rssd_index_ida); static int mtip_block_initialize(struct driver_data *dd); #ifdef CONFIG_COMPAT struct mtip_compat_ide_task_request_s { … }; #endif /* * This function check_for_surprise_removal is called * while card is removed from the system and it will * read the vendor id from the configuration space * * @pdev Pointer to the pci_dev structure. * * return value * true if device removed, else false */ static bool mtip_check_surprise_removal(struct driver_data *dd) { … } static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, unsigned int tag) { … } /* * Reset the HBA (without sleeping) * * @dd Pointer to the driver data structure. * * return value * 0 The reset was successful. * -1 The HBA Reset bit did not clear. */ static int mtip_hba_reset(struct driver_data *dd) { … } /* * Issue a command to the hardware. * * Set the appropriate bit in the s_active and Command Issue hardware * registers, causing hardware command processing to begin. * * @port Pointer to the port structure. * @tag The tag of the command to be issued. * * return value * None */ static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) { … } /* * Enable/disable the reception of FIS * * @port Pointer to the port data structure * @enable 1 to enable, 0 to disable * * return value * Previous state: 1 enabled, 0 disabled */ static int mtip_enable_fis(struct mtip_port *port, int enable) { … } /* * Enable/disable the DMA engine * * @port Pointer to the port data structure * @enable 1 to enable, 0 to disable * * return value * Previous state: 1 enabled, 0 disabled. */ static int mtip_enable_engine(struct mtip_port *port, int enable) { … } /* * Enables the port DMA engine and FIS reception. * * return value * None */ static inline void mtip_start_port(struct mtip_port *port) { … } /* * Deinitialize a port by disabling port interrupts, the DMA engine, * and FIS reception. * * @port Pointer to the port structure * * return value * None */ static inline void mtip_deinit_port(struct mtip_port *port) { … } /* * Initialize a port. * * This function deinitializes the port by calling mtip_deinit_port() and * then initializes it by setting the command header and RX FIS addresses, * clearing the SError register and any pending port interrupts before * re-enabling the default set of port interrupts. * * @port Pointer to the port structure. * * return value * None */ static void mtip_init_port(struct mtip_port *port) { … } /* * Restart a port * * @port Pointer to the port data structure. * * return value * None */ static void mtip_restart_port(struct mtip_port *port) { … } static int mtip_device_reset(struct driver_data *dd) { … } /* * Helper function for tag logging */ static void print_tags(struct driver_data *dd, char *msg, unsigned long *tagbits, int cnt) { … } static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, dma_addr_t buffer_dma, unsigned int sectors); static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, struct smart_attr *attrib); static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status) { … } /* * Handle an error. * * @dd Pointer to the DRIVER_DATA structure. * * return value * None */ static void mtip_handle_tfe(struct driver_data *dd) { … } /* * Handle a set device bits interrupt */ static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, u32 completed) { … } /* * Process legacy pio and d2h interrupts */ static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) { … } /* * Demux and handle errors */ static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) { … } static inline irqreturn_t mtip_handle_irq(struct driver_data *data) { … } /* * HBA interrupt subroutine. * * @irq IRQ number. * @instance Pointer to the driver data structure. * * return value * IRQ_HANDLED A HBA interrupt was pending and handled. * IRQ_NONE This interrupt was not for the HBA. */ static irqreturn_t mtip_irq_handler(int irq, void *instance) { … } static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) { … } static bool mtip_pause_ncq(struct mtip_port *port, struct host_to_dev_fis *fis) { … } static bool mtip_commands_active(struct mtip_port *port) { … } /* * Wait for port to quiesce * * @port Pointer to port data structure * @timeout Max duration to wait (ms) * * return value * 0 Success * -EBUSY Commands still active */ static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) { … } struct mtip_int_cmd { … }; /* * Execute an internal command and wait for the completion. * * @port Pointer to the port data structure. * @fis Pointer to the FIS that describes the command. * @fis_len Length in WORDS of the FIS. * @buffer DMA accessible for command data. * @buf_len Length, in bytes, of the data buffer. * @opts Command header options, excluding the FIS length * and the number of PRD entries. * @timeout Time in ms to wait for the command to complete. * * return value * 0 Command completed successfully. * -EFAULT The buffer address is not correctly aligned. * -EBUSY Internal command or other IO in progress. * -EAGAIN Time out waiting for command to complete. */ static int mtip_exec_internal_command(struct mtip_port *port, struct host_to_dev_fis *fis, int fis_len, dma_addr_t buffer, int buf_len, u32 opts, unsigned long timeout) { … } /* * Byte-swap ATA ID strings. * * ATA identify data contains strings in byte-swapped 16-bit words. * They must be swapped (on all architectures) to be usable as C strings. * This function swaps bytes in-place. * * @buf The buffer location of the string * @len The number of bytes to swap * * return value * None */ static inline void ata_swap_string(u16 *buf, unsigned int len) { … } static void mtip_set_timeout(struct driver_data *dd, struct host_to_dev_fis *fis, unsigned int *timeout, u8 erasemode) { … } /* * Request the device identity information. * * If a user space buffer is not specified, i.e. is NULL, the * identify information is still read from the drive and placed * into the identify data buffer (@e port->identify) in the * port data structure. * When the identify buffer contains valid identify information @e * port->identify_valid is non-zero. * * @port Pointer to the port structure. * @user_buffer A user space buffer where the identify data should be * copied. * * return value * 0 Command completed successfully. * -EFAULT An error occurred while coping data to the user buffer. * -1 Command failed. */ static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) { … } /* * Issue a standby immediate command to the device. * * @port Pointer to the port structure. * * return value * 0 Command was executed successfully. * -1 An error occurred while executing the command. */ static int mtip_standby_immediate(struct mtip_port *port) { … } /* * Issue a READ LOG EXT command to the device. * * @port pointer to the port structure. * @page page number to fetch * @buffer pointer to buffer * @buffer_dma dma address corresponding to @buffer * @sectors page length to fetch, in sectors * * return value * @rv return value from mtip_exec_internal_command() */ static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, dma_addr_t buffer_dma, unsigned int sectors) { … } /* * Issue a SMART READ DATA command to the device. * * @port pointer to the port structure. * @buffer pointer to buffer * @buffer_dma dma address corresponding to @buffer * * return value * @rv return value from mtip_exec_internal_command() */ static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, dma_addr_t buffer_dma) { … } /* * Get the value of a smart attribute * * @port pointer to the port structure * @id attribute number * @attrib pointer to return attrib information corresponding to @id * * return value * -EINVAL NULL buffer passed or unsupported attribute @id. * -EPERM Identify data not valid, SMART not supported or not enabled */ static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, struct smart_attr *attrib) { … } /* * Get the drive capacity. * * @dd Pointer to the device data structure. * @sectors Pointer to the variable that will receive the sector count. * * return value * 1 Capacity was returned successfully. * 0 The identify information is invalid. */ static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) { … } /* * Display the identify command data. * * @port Pointer to the port data structure. * * return value * None */ static void mtip_dump_identify(struct mtip_port *port) { … } /* * Map the commands scatter list into the command table. * * @command Pointer to the command. * @nents Number of scatter list entries. * * return value * None */ static inline void fill_command_sg(struct driver_data *dd, struct mtip_cmd *command, int nents) { … } /* * @brief Execute a drive command. * * return value 0 The command completed successfully. * return value -1 An error occurred while executing the command. */ static int exec_drive_task(struct mtip_port *port, u8 *command) { … } /* * @brief Execute a drive command. * * @param port Pointer to the port data structure. * @param command Pointer to the user specified command parameters. * @param user_buffer Pointer to the user space buffer where read sector * data should be copied. * * return value 0 The command completed successfully. * return value -EFAULT An error occurred while copying the completion * data to the user space buffer. * return value -1 An error occurred while executing the command. */ static int exec_drive_command(struct mtip_port *port, u8 *command, void __user *user_buffer) { … } /* * Indicates whether a command has a single sector payload. * * @command passed to the device to perform the certain event. * @features passed to the device to perform the certain event. * * return value * 1 command is one that always has a single sector payload, * regardless of the value in the Sector Count field. * 0 otherwise * */ static unsigned int implicit_sector(unsigned char command, unsigned char features) { … } /* * Executes a taskfile * See ide_taskfile_ioctl() for derivation */ static int exec_drive_taskfile(struct driver_data *dd, void __user *buf, ide_task_request_t *req_task, int outtotal) { … } /* * Handle IOCTL calls from the Block Layer. * * This function is called by the Block Layer when it receives an IOCTL * command that it does not understand. If the IOCTL command is not supported * this function returns -ENOTTY. * * @dd Pointer to the driver data structure. * @cmd IOCTL command passed from the Block Layer. * @arg IOCTL argument passed from the Block Layer. * * return value * 0 The IOCTL completed successfully. * -ENOTTY The specified command is not supported. * -EFAULT An error occurred copying data to a user space buffer. * -EIO An error occurred while executing the command. */ static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, unsigned long arg) { … } /* * Submit an IO to the hw * * This function is called by the block layer to issue an io * to the device. Upon completion, the callback function will * be called with the data parameter passed as the callback data. * * @dd Pointer to the driver data structure. * @start First sector to read. * @nsect Number of sectors to read. * @tag The tag of this read command. * @callback Pointer to the function that should be called * when the read completes. * @data Callback data passed to the callback function * when the read completes. * @dir Direction (read or write) * * return value * None */ static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, struct mtip_cmd *command, struct blk_mq_hw_ctx *hctx) { … } /* * Sysfs status dump. * * @dev Pointer to the device structure, passed by the kernrel. * @attr Pointer to the device_attribute structure passed by the kernel. * @buf Pointer to the char buffer that will receive the stats info. * * return value * The size, in bytes, of the data copied into buf. */ static ssize_t mtip_hw_show_status(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL); static struct attribute *mtip_disk_attrs[] = …; static const struct attribute_group mtip_disk_attr_group = …; static const struct attribute_group *mtip_disk_attr_groups[] = …; static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, size_t len, loff_t *offset) { … } static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, size_t len, loff_t *offset) { … } static const struct file_operations mtip_regs_fops = …; static const struct file_operations mtip_flags_fops = …; static void mtip_hw_debugfs_init(struct driver_data *dd) { … } static void mtip_hw_debugfs_exit(struct driver_data *dd) { … } /* * Perform any init/resume time hardware setup * * @dd Pointer to the driver data structure. * * return value * None */ static inline void hba_setup(struct driver_data *dd) { … } static int mtip_device_unaligned_constrained(struct driver_data *dd) { … } /* * Detect the details of the product, and store anything needed * into the driver data structure. This includes product type and * version and number of slot groups. * * @dd Pointer to the driver data structure. * * return value * None */ static void mtip_detect_product(struct driver_data *dd) { … } /* * Blocking wait for FTL rebuild to complete * * @dd Pointer to the DRIVER_DATA structure. * * return value * 0 FTL rebuild completed successfully * -EFAULT FTL rebuild error/timeout/interruption */ static int mtip_ftl_rebuild_poll(struct driver_data *dd) { … } static void mtip_softirq_done_fn(struct request *rq) { … } static bool mtip_abort_cmd(struct request *req, void *data) { … } static bool mtip_queue_cmd(struct request *req, void *data) { … } /* * service thread to issue queued commands * * @data Pointer to the driver data structure. * * return value * 0 */ static int mtip_service_thread(void *data) { … } /* * DMA region teardown * * @dd Pointer to driver_data structure * * return value * None */ static void mtip_dma_free(struct driver_data *dd) { … } /* * DMA region setup * * @dd Pointer to driver_data structure * * return value * -ENOMEM Not enough free DMA region space to initialize driver */ static int mtip_dma_alloc(struct driver_data *dd) { … } static int mtip_hw_get_identify(struct driver_data *dd) { … } /* * Called once for each card. * * @dd Pointer to the driver data structure. * * return value * 0 on success, else an error code. */ static int mtip_hw_init(struct driver_data *dd) { … } static int mtip_standby_drive(struct driver_data *dd) { … } /* * Called to deinitialize an interface. * * @dd Pointer to the driver data structure. * * return value * 0 */ static int mtip_hw_exit(struct driver_data *dd) { … } /* * Issue a Standby Immediate command to the device. * * This function is called by the Block Layer just before the * system powers off during a shutdown. * * @dd Pointer to the driver data structure. * * return value * 0 */ static int mtip_hw_shutdown(struct driver_data *dd) { … } /* * Suspend function * * This function is called by the Block Layer just before the * system hibernates. * * @dd Pointer to the driver data structure. * * return value * 0 Suspend was successful * -EFAULT Suspend was not successful */ static int mtip_hw_suspend(struct driver_data *dd) { … } /* * Resume function * * This function is called by the Block Layer as the * system resumes. * * @dd Pointer to the driver data structure. * * return value * 0 Resume was successful * -EFAULT Resume was not successful */ static int mtip_hw_resume(struct driver_data *dd) { … } /* * Helper function for reusing disk name * upon hot insertion. */ static int rssd_disk_name_format(char *prefix, int index, char *buf, int buflen) { … } /* * Block layer IOCTL handler. * * @dev Pointer to the block_device structure. * @mode ignored * @cmd IOCTL command passed from the user application. * @arg Argument passed from the user application. * * return value * 0 IOCTL completed successfully. * -ENOTTY IOCTL not supported or invalid driver data * structure pointer. */ static int mtip_block_ioctl(struct block_device *dev, blk_mode_t mode, unsigned cmd, unsigned long arg) { … } #ifdef CONFIG_COMPAT /* * Block layer compat IOCTL handler. * * @dev Pointer to the block_device structure. * @mode ignored * @cmd IOCTL command passed from the user application. * @arg Argument passed from the user application. * * return value * 0 IOCTL completed successfully. * -ENOTTY IOCTL not supported or invalid driver data * structure pointer. */ static int mtip_block_compat_ioctl(struct block_device *dev, blk_mode_t mode, unsigned cmd, unsigned long arg) { … } #endif /* * Obtain the geometry of the device. * * You may think that this function is obsolete, but some applications, * fdisk for example still used CHS values. This function describes the * device as having 224 heads and 56 sectors per cylinder. These values are * chosen so that each cylinder is aligned on a 4KB boundary. Since a * partition is described in terms of a start and end cylinder this means * that each partition is also 4KB aligned. Non-aligned partitions adversely * affects performance. * * @dev Pointer to the block_device strucutre. * @geo Pointer to a hd_geometry structure. * * return value * 0 Operation completed successfully. * -ENOTTY An error occurred while reading the drive capacity. */ static int mtip_block_getgeo(struct block_device *dev, struct hd_geometry *geo) { … } static void mtip_block_free_disk(struct gendisk *disk) { … } /* * Block device operation function. * * This structure contains pointers to the functions required by the block * layer. */ static const struct block_device_operations mtip_block_ops = …; static inline bool is_se_active(struct driver_data *dd) { … } static inline bool is_stopped(struct driver_data *dd, struct request *rq) { … } static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, struct request *rq) { … } static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx, struct request *rq) { … } static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { … } static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx) { … } static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) { … } static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req) { … } static const struct blk_mq_ops mtip_mq_ops = …; /* * Block layer initialization function. * * This function is called once by the PCI layer for each P320 * device that is connected to the system. * * @dd Pointer to the driver data structure. * * return value * 0 on success else an error code. */ static int mtip_block_initialize(struct driver_data *dd) { … } /* * Function called by the PCI layer when just before the * machine shuts down. * * If a protocol layer shutdown function is present it will be called * by this function. * * @dd Pointer to the driver data structure. * * return value * 0 */ static int mtip_block_shutdown(struct driver_data *dd) { … } static int mtip_block_suspend(struct driver_data *dd) { … } static int mtip_block_resume(struct driver_data *dd) { … } static void drop_cpu(int cpu) { … } static int get_least_used_cpu_on_node(int node) { … } /* Helper for selecting a node in round robin mode */ static inline int mtip_get_next_rr_node(void) { … } static DEFINE_HANDLER(0); static DEFINE_HANDLER(1); static DEFINE_HANDLER(2); static DEFINE_HANDLER(3); static DEFINE_HANDLER(4); static DEFINE_HANDLER(5); static DEFINE_HANDLER(6); static DEFINE_HANDLER(7); static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) { … } static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) { … } /* * Called for each supported PCI device detected. * * This function allocates the private data structure, enables the * PCI device and then calls the block layer initialization function. * * return value * 0 on success else an error code. */ static int mtip_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { … } /* * Called for each probed device when the device is removed or the * driver is unloaded. * * return value * None */ static void mtip_pci_remove(struct pci_dev *pdev) { … } /* * Called for each probed device when the device is suspended. * * return value * 0 Success * <0 Error */ static int __maybe_unused mtip_pci_suspend(struct device *dev) { … } /* * Called for each probed device when the device is resumed. * * return value * 0 Success * <0 Error */ static int __maybe_unused mtip_pci_resume(struct device *dev) { … } /* * Shutdown routine * * return value * None */ static void mtip_pci_shutdown(struct pci_dev *pdev) { … } /* Table of device ids supported by this driver. */ static const struct pci_device_id mtip_pci_tbl[] = …; static SIMPLE_DEV_PM_OPS(mtip_pci_pm_ops, mtip_pci_suspend, mtip_pci_resume); /* Structure that describes the PCI driver functions. */ static struct pci_driver mtip_pci_driver = …; MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); /* * Module initialization function. * * Called once when the module is loaded. This function allocates a major * block device number to the Cyclone devices and registers the PCI layer * of the driver. * * Return value * 0 on success else error code. */ static int __init mtip_init(void) { … } /* * Module de-initialization function. * * Called once when the module is unloaded. This function deallocates * the major block device number allocated by mtip_init() and * unregisters the PCI layer of the driver. * * Return value * none */ static void __exit mtip_exit(void) { … } MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_VERSION(…); module_init(…) …; module_exit(mtip_exit);
Codebase Browser
linux