// SPDX-License-Identifier: GPL-2.0+ /* * Cadence NAND flash controller driver * * Copyright (C) 2019 Cadence * * Author: Piotr Sroka <[email protected]> */ #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/mtd/mtd.h> #include <linux/mtd/rawnand.h> #include <linux/iopoll.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/property.h> #include <linux/slab.h> /* * HPNFC can work in 3 modes: * - PIO - can work in master or slave DMA * - CDMA - needs Master DMA for accessing command descriptors. * - Generic mode - can use only slave DMA. * CDMA and PIO modes can be used to execute only base commands. * Generic mode can be used to execute any command * on NAND flash memory. Driver uses CDMA mode for * block erasing, page reading, page programing. * Generic mode is used for executing rest of commands. */ #define MAX_ADDRESS_CYC … #define MAX_ERASE_ADDRESS_CYC … #define MAX_DATA_SIZE … #define DMA_DATA_SIZE_ALIGN … /* Register definition. */ /* * Command register 0. * Writing data to this register will initiate a new transaction * of the NF controller. */ #define CMD_REG0 … /* Command type field mask. */ #define CMD_REG0_CT … /* Command type CDMA. */ #define CMD_REG0_CT_CDMA … /* Command type generic. */ #define CMD_REG0_CT_GEN … /* Command thread number field mask. */ #define CMD_REG0_TN … /* Command register 2. */ #define CMD_REG2 … /* Command register 3. */ #define CMD_REG3 … /* Pointer register to select which thread status will be selected. */ #define CMD_STATUS_PTR … /* Command status register for selected thread. */ #define CMD_STATUS … /* Interrupt status register. */ #define INTR_STATUS … #define INTR_STATUS_SDMA_ERR … #define INTR_STATUS_SDMA_TRIGG … #define INTR_STATUS_UNSUPP_CMD … #define INTR_STATUS_DDMA_TERR … #define INTR_STATUS_CDMA_TERR … #define INTR_STATUS_CDMA_IDL … /* Interrupt enable register. */ #define INTR_ENABLE … #define INTR_ENABLE_INTR_EN … #define INTR_ENABLE_SDMA_ERR_EN … #define INTR_ENABLE_SDMA_TRIGG_EN … #define INTR_ENABLE_UNSUPP_CMD_EN … #define INTR_ENABLE_DDMA_TERR_EN … #define INTR_ENABLE_CDMA_TERR_EN … #define INTR_ENABLE_CDMA_IDLE_EN … /* Controller internal state. */ #define CTRL_STATUS … #define CTRL_STATUS_INIT_COMP … #define CTRL_STATUS_CTRL_BUSY … /* Command Engine threads state. */ #define TRD_STATUS … /* Command Engine interrupt thread error status. */ #define TRD_ERR_INT_STATUS … /* Command Engine interrupt thread error enable. */ #define TRD_ERR_INT_STATUS_EN … /* Command Engine interrupt thread complete status. */ #define TRD_COMP_INT_STATUS … /* * Transfer config 0 register. * Configures data transfer parameters. */ #define TRAN_CFG_0 … /* Offset value from the beginning of the page. */ #define TRAN_CFG_0_OFFSET … /* Numbers of sectors to transfer within singlNF device's page. */ #define TRAN_CFG_0_SEC_CNT … /* * Transfer config 1 register. * Configures data transfer parameters. */ #define TRAN_CFG_1 … /* Size of last data sector. */ #define TRAN_CFG_1_LAST_SEC_SIZE … /* Size of not-last data sector. */ #define TRAN_CFG_1_SECTOR_SIZE … /* ECC engine configuration register 0. */ #define ECC_CONFIG_0 … /* Correction strength. */ #define ECC_CONFIG_0_CORR_STR … /* Enable erased pages detection mechanism. */ #define ECC_CONFIG_0_ERASE_DET_EN … /* Enable controller ECC check bits generation and correction. */ #define ECC_CONFIG_0_ECC_EN … /* ECC engine configuration register 1. */ #define ECC_CONFIG_1 … /* Multiplane settings register. */ #define MULTIPLANE_CFG … /* Cache operation settings. */ #define CACHE_CFG … /* DMA settings register. */ #define DMA_SETINGS … /* Enable SDMA error report on access unprepared slave DMA interface. */ #define DMA_SETINGS_SDMA_ERR_RSP … /* Transferred data block size for the slave DMA module. */ #define SDMA_SIZE … /* Thread number associated with transferred data block * for the slave DMA module. */ #define SDMA_TRD_NUM … /* Thread number mask. */ #define SDMA_TRD_NUM_SDMA_TRD … #define CONTROL_DATA_CTRL … /* Thread number mask. */ #define CONTROL_DATA_CTRL_SIZE … #define CTRL_VERSION … #define CTRL_VERSION_REV … /* Available hardware features of the controller. */ #define CTRL_FEATURES … /* Support for NV-DDR2/3 work mode. */ #define CTRL_FEATURES_NVDDR_2_3 … /* Support for NV-DDR work mode. */ #define CTRL_FEATURES_NVDDR … /* Support for asynchronous work mode. */ #define CTRL_FEATURES_ASYNC … /* Support for asynchronous work mode. */ #define CTRL_FEATURES_N_BANKS … /* Slave and Master DMA data width. */ #define CTRL_FEATURES_DMA_DWITH64 … /* Availability of Control Data feature.*/ #define CTRL_FEATURES_CONTROL_DATA … /* BCH Engine identification register 0 - correction strengths. */ #define BCH_CFG_0 … #define BCH_CFG_0_CORR_CAP_0 … #define BCH_CFG_0_CORR_CAP_1 … #define BCH_CFG_0_CORR_CAP_2 … #define BCH_CFG_0_CORR_CAP_3 … /* BCH Engine identification register 1 - correction strengths. */ #define BCH_CFG_1 … #define BCH_CFG_1_CORR_CAP_4 … #define BCH_CFG_1_CORR_CAP_5 … #define BCH_CFG_1_CORR_CAP_6 … #define BCH_CFG_1_CORR_CAP_7 … /* BCH Engine identification register 2 - sector sizes. */ #define BCH_CFG_2 … #define BCH_CFG_2_SECT_0 … #define BCH_CFG_2_SECT_1 … /* BCH Engine identification register 3. */ #define BCH_CFG_3 … #define BCH_CFG_3_METADATA_SIZE … /* Ready/Busy# line status. */ #define RBN_SETINGS … /* Common settings. */ #define COMMON_SET … /* 16 bit device connected to the NAND Flash interface. */ #define COMMON_SET_DEVICE_16BIT … /* Skip_bytes registers. */ #define SKIP_BYTES_CONF … #define SKIP_BYTES_MARKER_VALUE … #define SKIP_BYTES_NUM_OF_BYTES … #define SKIP_BYTES_OFFSET … #define SKIP_BYTES_OFFSET_VALUE … /* Timings configuration. */ #define ASYNC_TOGGLE_TIMINGS … #define ASYNC_TOGGLE_TIMINGS_TRH … #define ASYNC_TOGGLE_TIMINGS_TRP … #define ASYNC_TOGGLE_TIMINGS_TWH … #define ASYNC_TOGGLE_TIMINGS_TWP … #define TIMINGS0 … #define TIMINGS0_TADL … #define TIMINGS0_TCCS … #define TIMINGS0_TWHR … #define TIMINGS0_TRHW … #define TIMINGS1 … #define TIMINGS1_TRHZ … #define TIMINGS1_TWB … #define TIMINGS1_TVDLY … #define TIMINGS2 … #define TIMINGS2_TFEAT … #define TIMINGS2_CS_HOLD_TIME … #define TIMINGS2_CS_SETUP_TIME … /* Configuration of the resynchronization of slave DLL of PHY. */ #define DLL_PHY_CTRL … #define DLL_PHY_CTRL_DLL_RST_N … #define DLL_PHY_CTRL_EXTENDED_WR_MODE … #define DLL_PHY_CTRL_EXTENDED_RD_MODE … #define DLL_PHY_CTRL_RS_HIGH_WAIT_CNT … #define DLL_PHY_CTRL_RS_IDLE_CNT … /* Register controlling DQ related timing. */ #define PHY_DQ_TIMING … /* Register controlling DSQ related timing. */ #define PHY_DQS_TIMING … #define PHY_DQS_TIMING_DQS_SEL_OE_END … #define PHY_DQS_TIMING_PHONY_DQS_SEL … #define PHY_DQS_TIMING_USE_PHONY_DQS … /* Register controlling the gate and loopback control related timing. */ #define PHY_GATE_LPBK_CTRL … #define PHY_GATE_LPBK_CTRL_RDS … /* Register holds the control for the master DLL logic. */ #define PHY_DLL_MASTER_CTRL … #define PHY_DLL_MASTER_CTRL_BYPASS_MODE … /* Register holds the control for the slave DLL logic. */ #define PHY_DLL_SLAVE_CTRL … /* This register handles the global control settings for the PHY. */ #define PHY_CTRL … #define PHY_CTRL_SDR_DQS … #define PHY_CTRL_PHONY_DQS … /* * This register handles the global control settings * for the termination selects for reads. */ #define PHY_TSEL … /* Generic command layout. */ #define GCMD_LAY_CS … /* * This bit informs the minicotroller if it has to wait for tWB * after sending the last CMD/ADDR/DATA in the sequence. */ #define GCMD_LAY_TWB … /* Type of generic instruction. */ #define GCMD_LAY_INSTR … /* Generic CMD sequence type. */ #define GCMD_LAY_INSTR_CMD … /* Generic ADDR sequence type. */ #define GCMD_LAY_INSTR_ADDR … /* Generic data transfer sequence type. */ #define GCMD_LAY_INSTR_DATA … /* Input part of generic command type of input is command. */ #define GCMD_LAY_INPUT_CMD … /* Generic command address sequence - address fields. */ #define GCMD_LAY_INPUT_ADDR … /* Generic command address sequence - address size. */ #define GCMD_LAY_INPUT_ADDR_SIZE … /* Transfer direction field of generic command data sequence. */ #define GCMD_DIR … /* Read transfer direction of generic command data sequence. */ #define GCMD_DIR_READ … /* Write transfer direction of generic command data sequence. */ #define GCMD_DIR_WRITE … /* ECC enabled flag of generic command data sequence - ECC enabled. */ #define GCMD_ECC_EN … /* Generic command data sequence - sector size. */ #define GCMD_SECT_SIZE … /* Generic command data sequence - sector count. */ #define GCMD_SECT_CNT … /* Generic command data sequence - last sector size. */ #define GCMD_LAST_SIZE … /* CDMA descriptor fields. */ /* Erase command type of CDMA descriptor. */ #define CDMA_CT_ERASE … /* Program page command type of CDMA descriptor. */ #define CDMA_CT_WR … /* Read page command type of CDMA descriptor. */ #define CDMA_CT_RD … /* Flash pointer memory shift. */ #define CDMA_CFPTR_MEM_SHIFT … /* Flash pointer memory mask. */ #define CDMA_CFPTR_MEM … /* * Command DMA descriptor flags. If set causes issue interrupt after * the completion of descriptor processing. */ #define CDMA_CF_INT … /* * Command DMA descriptor flags - the next descriptor * address field is valid and descriptor processing should continue. */ #define CDMA_CF_CONT … /* DMA master flag of command DMA descriptor. */ #define CDMA_CF_DMA_MASTER … /* Operation complete status of command descriptor. */ #define CDMA_CS_COMP … /* Operation complete status of command descriptor. */ /* Command descriptor status - operation fail. */ #define CDMA_CS_FAIL … /* Command descriptor status - page erased. */ #define CDMA_CS_ERP … /* Command descriptor status - timeout occurred. */ #define CDMA_CS_TOUT … /* * Maximum amount of correction applied to one ECC sector. * It is part of command descriptor status. */ #define CDMA_CS_MAXERR … /* Command descriptor status - uncorrectable ECC error. */ #define CDMA_CS_UNCE … /* Command descriptor status - descriptor error. */ #define CDMA_CS_ERR … /* Status of operation - OK. */ #define STAT_OK … /* Status of operation - FAIL. */ #define STAT_FAIL … /* Status of operation - uncorrectable ECC error. */ #define STAT_ECC_UNCORR … /* Status of operation - page erased. */ #define STAT_ERASED … /* Status of operation - correctable ECC error. */ #define STAT_ECC_CORR … /* Status of operation - unsuspected state. */ #define STAT_UNKNOWN … /* Status of operation - operation is not completed yet. */ #define STAT_BUSY … #define BCH_MAX_NUM_CORR_CAPS … #define BCH_MAX_NUM_SECTOR_SIZES … struct cadence_nand_timings { … }; /* Command DMA descriptor. */ struct cadence_nand_cdma_desc { … }; /* Interrupt status. */ struct cadence_nand_irq_status { … }; /* Cadence NAND flash controller capabilities get from driver data. */ struct cadence_nand_dt_devdata { … }; /* Cadence NAND flash controller capabilities read from registers. */ struct cdns_nand_caps { … }; struct cdns_nand_ctrl { … }; struct cdns_nand_chip { … }; static inline struct cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip) { … } static inline struct cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller) { … } static bool cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf, u32 buf_len) { … } static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl, u32 reg_offset, u32 timeout_us, u32 mask, bool is_clear) { … } static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl, bool enable) { … } static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl, u8 corr_str_idx) { … } static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl, u8 strength) { … } static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl, u16 marker_value) { … } static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl, u8 num_of_bytes, u32 offset_value, int enable) { … } /* Functions enables/disables hardware detection of erased data */ static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl, bool enable, u8 bitflips_threshold) { … } static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl, bool bit_bus16) { … } static void cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_irq_status *irq_status) { … } static void cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_irq_status *irq_status) { … } static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_irq_status *irq_status) { … } static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl) { … } /* * This is the interrupt service routine. It handles all interrupts * sent to this device. */ static irqreturn_t cadence_nand_isr(int irq, void *dev_id) { … } static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_irq_status *irq_mask) { … } static void cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_irq_status *irq_mask, struct cadence_nand_irq_status *irq_status) { … } /* Execute generic command on NAND controller. */ static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr, u64 mini_ctrl_cmd) { … } /* Wait for data on slave DMA interface. */ static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl, u8 *out_sdma_trd, u32 *out_sdma_size) { … } static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl) { … } /* Prepare CDMA descriptor. */ static void cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl, char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr, dma_addr_t ctrl_data_ptr, u16 ctype) { … } static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl, u32 desc_status) { … } static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl) { … } static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { … } /* Send SDMA command and wait for finish. */ static u32 cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { … } /* * ECC size depends on configured ECC strength and on maximum supported * ECC step size. */ static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength) { … } #define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) … CADENCE_NAND_CALC_ECC_BYTES(…) CADENCE_NAND_CALC_ECC_BYTES(…) CADENCE_NAND_CALC_ECC_BYTES(…) CADENCE_NAND_CALC_ECC_BYTES(…) CADENCE_NAND_CALC_ECC_BYTES(…) /* Function reads BCH capabilities. */ static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl) { … } /* Hardware initialization. */ static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl) { … } #define TT_MAIN_OOB_AREAS … #define TT_RAW_PAGE … #define TT_BBM … #define TT_MAIN_OOB_AREA_EXT … /* Prepare size of data to transfer. */ static void cadence_nand_prepare_data_size(struct nand_chip *chip, int transfer_type) { … } static int cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr, int page, void *buf, void *ctrl_dat, u32 buf_size, u32 ctrl_dat_size, enum dma_data_direction dir, bool with_ecc) { … } static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl, struct cadence_nand_timings *t) { … } static int cadence_nand_select_target(struct nand_chip *chip) { … } static int cadence_nand_erase(struct nand_chip *chip, u32 page) { … } static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf) { … } static int cadence_nand_write_page(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { … } static int cadence_nand_write_oob(struct nand_chip *chip, int page) { … } static int cadence_nand_write_page_raw(struct nand_chip *chip, const u8 *buf, int oob_required, int page) { … } static int cadence_nand_write_oob_raw(struct nand_chip *chip, int page) { … } static int cadence_nand_read_page(struct nand_chip *chip, u8 *buf, int oob_required, int page) { … } /* Reads OOB data from the device. */ static int cadence_nand_read_oob(struct nand_chip *chip, int page) { … } static int cadence_nand_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_required, int page) { … } static int cadence_nand_read_oob_raw(struct nand_chip *chip, int page) { … } static void cadence_nand_slave_dma_transfer_finished(void *data) { … } static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl, void *buf, dma_addr_t dev_dma, size_t len, enum dma_data_direction dir) { … } static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl, u8 *buf, int len) { … } static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl, const u8 *buf, int len) { … } static int cadence_nand_force_byte_access(struct nand_chip *chip, bool force_8bit) { … } static int cadence_nand_cmd_opcode(struct nand_chip *chip, const struct nand_subop *subop) { … } static int cadence_nand_cmd_address(struct nand_chip *chip, const struct nand_subop *subop) { … } static int cadence_nand_cmd_erase(struct nand_chip *chip, const struct nand_subop *subop) { … } static int cadence_nand_cmd_data(struct nand_chip *chip, const struct nand_subop *subop) { … } static int cadence_nand_cmd_waitrdy(struct nand_chip *chip, const struct nand_subop *subop) { … } static const struct nand_op_parser cadence_nand_op_parser = …; static int cadence_nand_exec_op(struct nand_chip *chip, const struct nand_operation *op, bool check_only) { … } static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section, struct mtd_oob_region *oobregion) { … } static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section, struct mtd_oob_region *oobregion) { … } static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = …; static int calc_cycl(u32 timing, u32 clock) { … } /* Calculate max data valid window. */ static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min, u32 board_delay_skew_min, u32 ext_mode) { … } /* Calculate data valid window. */ static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min, u32 trea_max, u32 ext_mode) { … } static int cadence_nand_setup_interface(struct nand_chip *chip, int chipnr, const struct nand_interface_config *conf) { … } static int cadence_nand_attach_chip(struct nand_chip *chip) { … } static const struct nand_controller_ops cadence_nand_controller_ops = …; static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl, struct device_node *np) { … } static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl) { … } static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl) { … } static void cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl) { … } static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) { … } /* Driver exit point. */ static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl) { … } struct cadence_nand_dt { … }; static const struct cadence_nand_dt_devdata cadence_nand_default = …; static const struct of_device_id cadence_nand_dt_ids[] = …; MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids); static int cadence_nand_dt_probe(struct platform_device *ofdev) { … } static void cadence_nand_dt_remove(struct platform_device *ofdev) { … } static struct platform_driver cadence_nand_dt_driver = …; module_platform_driver(…) …; MODULE_AUTHOR(…) …; MODULE_LICENSE(…) …; MODULE_DESCRIPTION(…) …;
Codebase Browser
linux