// SPDX-License-Identifier: GPL-2.0-only /* * Intel Keem Bay OCS AES Crypto Driver. * * Copyright (C) 2018-2020 Intel Corporation */ #include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/swab.h> #include <asm/byteorder.h> #include <asm/errno.h> #include <crypto/aes.h> #include <crypto/gcm.h> #include "ocs-aes.h" #define AES_COMMAND_OFFSET … #define AES_KEY_0_OFFSET … #define AES_KEY_1_OFFSET … #define AES_KEY_2_OFFSET … #define AES_KEY_3_OFFSET … #define AES_KEY_4_OFFSET … #define AES_KEY_5_OFFSET … #define AES_KEY_6_OFFSET … #define AES_KEY_7_OFFSET … #define AES_IV_0_OFFSET … #define AES_IV_1_OFFSET … #define AES_IV_2_OFFSET … #define AES_IV_3_OFFSET … #define AES_ACTIVE_OFFSET … #define AES_STATUS_OFFSET … #define AES_KEY_SIZE_OFFSET … #define AES_IER_OFFSET … #define AES_ISR_OFFSET … #define AES_MULTIPURPOSE1_0_OFFSET … #define AES_MULTIPURPOSE1_1_OFFSET … #define AES_MULTIPURPOSE1_2_OFFSET … #define AES_MULTIPURPOSE1_3_OFFSET … #define AES_MULTIPURPOSE2_0_OFFSET … #define AES_MULTIPURPOSE2_1_OFFSET … #define AES_MULTIPURPOSE2_2_OFFSET … #define AES_MULTIPURPOSE2_3_OFFSET … #define AES_BYTE_ORDER_CFG_OFFSET … #define AES_TLEN_OFFSET … #define AES_T_MAC_0_OFFSET … #define AES_T_MAC_1_OFFSET … #define AES_T_MAC_2_OFFSET … #define AES_T_MAC_3_OFFSET … #define AES_PLEN_OFFSET … #define AES_A_DMA_SRC_ADDR_OFFSET … #define AES_A_DMA_DST_ADDR_OFFSET … #define AES_A_DMA_SRC_SIZE_OFFSET … #define AES_A_DMA_DST_SIZE_OFFSET … #define AES_A_DMA_DMA_MODE_OFFSET … #define AES_A_DMA_NEXT_SRC_DESCR_OFFSET … #define AES_A_DMA_NEXT_DST_DESCR_OFFSET … #define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET … #define AES_A_DMA_LOG_OFFSET … #define AES_A_DMA_STATUS_OFFSET … #define AES_A_DMA_PERF_CNTR_OFFSET … #define AES_A_DMA_MSI_ISR_OFFSET … #define AES_A_DMA_MSI_IER_OFFSET … #define AES_A_DMA_MSI_MASK_OFFSET … #define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET … #define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET … /* * AES_A_DMA_DMA_MODE register. * Default: 0x00000000. * bit[31] ACTIVE * This bit activates the DMA. When the DMA finishes, it resets * this bit to zero. * bit[30:26] Unused by this driver. * bit[25] SRC_LINK_LIST_EN * Source link list enable bit. When the linked list is terminated * this bit is reset by the DMA. * bit[24] DST_LINK_LIST_EN * Destination link list enable bit. When the linked list is * terminated this bit is reset by the DMA. * bit[23:0] Unused by this driver. */ #define AES_A_DMA_DMA_MODE_ACTIVE … #define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN … #define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN … /* * AES_ACTIVE register * default 0x00000000 * bit[31:10] Reserved * bit[9] LAST_ADATA * bit[8] LAST_GCX * bit[7:2] Reserved * bit[1] TERMINATION * bit[0] TRIGGER */ #define AES_ACTIVE_LAST_ADATA … #define AES_ACTIVE_LAST_CCM_GCM … #define AES_ACTIVE_TERMINATION … #define AES_ACTIVE_TRIGGER … #define AES_DISABLE_INT … #define AES_DMA_CPD_ERR_INT … #define AES_DMA_OUTBUF_RD_ERR_INT … #define AES_DMA_OUTBUF_WR_ERR_INT … #define AES_DMA_INBUF_RD_ERR_INT … #define AES_DMA_INBUF_WR_ERR_INT … #define AES_DMA_BAD_COMP_INT … #define AES_DMA_SAI_INT … #define AES_DMA_SRC_DONE_INT … #define AES_COMPLETE_INT … #define AES_DMA_MSI_MASK_CLEAR … #define AES_128_BIT_KEY … #define AES_256_BIT_KEY … #define AES_DEACTIVATE_PERF_CNTR … #define AES_ACTIVATE_PERF_CNTR … #define AES_MAX_TAG_SIZE_U32 … #define OCS_LL_DMA_FLAG_TERMINATE … /* * There is an inconsistency in the documentation. This is documented as a * 11-bit value, but it is actually 10-bits. */ #define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK … /* * During CCM decrypt, the OCS block needs to finish processing the ciphertext * before the tag is written. For 128-bit mode this required delay is 28 OCS * clock cycles. For 256-bit mode it is 36 OCS clock cycles. */ #define CCM_DECRYPT_DELAY_TAG_CLK_COUNT … /* * During CCM decrypt there must be a delay of at least 42 OCS clock cycles * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM * bit in the same register (as stated in the OCS databook) */ #define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT … /* See RFC3610 section 2.2 */ #define L_PRIME_MIN … #define L_PRIME_MAX … /* * CCM IV format from RFC 3610 section 2.3 * * Octet Number Contents * ------------ --------- * 0 Flags * 1 ... 15-L Nonce N * 16-L ... 15 Counter i * * Flags = L' = L - 1 */ #define L_PRIME_IDX … #define COUNTER_START(lprime) … #define COUNTER_LEN(lprime) … enum aes_counter_mode { … }; /** * struct ocs_dma_linked_list - OCS DMA linked list entry. * @src_addr: Source address of the data. * @src_len: Length of data to be fetched. * @next: Next dma_list to fetch. * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. */ struct ocs_dma_linked_list { … } __packed; /* * Set endianness of inputs and outputs * AES_BYTE_ORDER_CFG * default 0x00000000 * bit [10] - KEY_HI_LO_SWAP * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD * bit [4] - IV_SWAP_BYTES_IN_DWORD * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD * bit [2] - DOUT_SWAP_BYTES_IN_DWORD * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD * bit [0] - DOUT_SWAP_BYTES_IN_DWORD */ static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) { … } /* Trigger AES process start. */ static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) { … } /* Indicate last bulk of data. */ static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) { … } /* * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. * * Called when DMA is programmed to fetch the last batch of data. * - For AES-CCM it is called for the last batch of Payload data and Ciphertext * data. * - For AES-GCM, it is called for the last batch of Plaintext data and * Ciphertext data. */ static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) { … } /* Wait for LAST_CCM_GCM bit to be unset. */ static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) { … } /* Wait for 10 bits of input occupancy. */ static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) { … } /* * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all * other bits). * * Called when DMA is programmed to fetch the last batch of Associated Data * (CCM case) or Additional Authenticated Data (GCM case). */ static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) { … } /* Set DMA src and dst transfer size to 0 */ static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) { … } /* Activate DMA for zero-byte transfer case. */ static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) { … } /* Activate DMA and enable src linked list */ static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) { … } /* Activate DMA and enable dst linked list */ static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) { … } /* Activate DMA and enable src and dst linked lists */ static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) { … } /* Reset PERF_CNTR to 0 and activate it */ static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) { … } /* Wait until PERF_CNTR is > delay, then deactivate it */ static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, int delay) { … } /* Disable AES and DMA IRQ. */ static void aes_irq_disable(struct ocs_aes_dev *aes_dev) { … } /* Enable AES or DMA IRQ. IRQ is disabled once fired. */ static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) { … } /* Enable and wait for IRQ (either from OCS AES engine or DMA) */ static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) { … } /* Configure DMA to OCS, linked list mode */ static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, dma_addr_t dma_list) { … } /* Configure DMA from OCS, linked list mode */ static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, dma_addr_t dma_list) { … } irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) { … } /** * ocs_aes_set_key() - Write key into OCS AES hardware. * @aes_dev: The OCS AES device to write the key to. * @key_size: The size of the key (in bytes). * @key: The key to write. * @cipher: The cipher the key is for. * * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. * * Return: 0 on success, negative error code otherwise. */ int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, enum ocs_cipher cipher) { … } /* Write AES_COMMAND */ static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, enum ocs_cipher cipher, enum ocs_mode mode, enum ocs_instruction instruction) { … } static void ocs_aes_init(struct ocs_aes_dev *aes_dev, enum ocs_mode mode, enum ocs_cipher cipher, enum ocs_instruction instruction) { … } /* * Write the byte length of the last AES/SM4 block of Payload data (without * zero padding and without the length of the MAC) in register AES_PLEN. */ static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, u32 size) { … } /* * Validate inputs according to mode. * If OK return 0; else return -EINVAL. */ static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, const u8 *iv, u32 iv_size, dma_addr_t aad_dma_list, u32 aad_size, const u8 *tag, u32 tag_size, enum ocs_cipher cipher, enum ocs_mode mode, enum ocs_instruction instruction, dma_addr_t dst_dma_list) { … } /** * ocs_aes_op() - Perform AES/SM4 operation. * @aes_dev: The OCS AES device to use. * @mode: The mode to use (ECB, CBC, CTR, or CTS). * @cipher: The cipher to use (AES or SM4). * @instruction: The instruction to perform (encrypt or decrypt). * @dst_dma_list: The OCS DMA list mapping output memory. * @src_dma_list: The OCS DMA list mapping input payload data. * @src_size: The amount of data mapped by @src_dma_list. * @iv: The IV vector. * @iv_size: The size (in bytes) of @iv. * * Return: 0 on success, negative error code otherwise. */ int ocs_aes_op(struct ocs_aes_dev *aes_dev, enum ocs_mode mode, enum ocs_cipher cipher, enum ocs_instruction instruction, dma_addr_t dst_dma_list, dma_addr_t src_dma_list, u32 src_size, u8 *iv, u32 iv_size) { … } /* Compute and write J0 to engine registers. */ static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, const u8 *iv) { … } /* Read GCM tag from engine registers. */ static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, u8 *tag, u32 tag_size) { … } /** * ocs_aes_gcm_op() - Perform GCM operation. * @aes_dev: The OCS AES device to use. * @cipher: The Cipher to use (AES or SM4). * @instruction: The instruction to perform (encrypt or decrypt). * @dst_dma_list: The OCS DMA list mapping output memory. * @src_dma_list: The OCS DMA list mapping input payload data. * @src_size: The amount of data mapped by @src_dma_list. * @iv: The input IV vector. * @aad_dma_list: The OCS DMA list mapping input AAD data. * @aad_size: The amount of data mapped by @aad_dma_list. * @out_tag: Where to store computed tag. * @tag_size: The size (in bytes) of @out_tag. * * Return: 0 on success, negative error code otherwise. */ int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, enum ocs_cipher cipher, enum ocs_instruction instruction, dma_addr_t dst_dma_list, dma_addr_t src_dma_list, u32 src_size, const u8 *iv, dma_addr_t aad_dma_list, u32 aad_size, u8 *out_tag, u32 tag_size) { … } /* Write encrypted tag to AES/SM4 engine. */ static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, const u8 *in_tag, u32 tag_size) { … } /* * Write B0 CCM block to OCS AES HW. * * Note: B0 format is documented in NIST Special Publication 800-38C * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf * (see Section A.2.1) */ static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, const u8 *iv, u32 adata_size, u32 tag_size, u32 cryptlen) { … } /* * Write adata length to OCS AES HW. * * Note: adata len encoding is documented in NIST Special Publication 800-38C * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf * (see Section A.2.2) */ static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, u64 adata_len) { … } static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, dma_addr_t adata_dma_list, u32 adata_size) { … } static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, dma_addr_t dst_dma_list, dma_addr_t src_dma_list, u32 src_size) { … } static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, dma_addr_t dst_dma_list, dma_addr_t src_dma_list, u32 src_size) { … } /* * Compare Tag to Yr. * * Only used at the end of CCM decrypt. If tag == yr, message authentication * has succeeded. */ static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, u8 tag_size_bytes) { … } /** * ocs_aes_ccm_op() - Perform CCM operation. * @aes_dev: The OCS AES device to use. * @cipher: The Cipher to use (AES or SM4). * @instruction: The instruction to perform (encrypt or decrypt). * @dst_dma_list: The OCS DMA list mapping output memory. * @src_dma_list: The OCS DMA list mapping input payload data. * @src_size: The amount of data mapped by @src_dma_list. * @iv: The input IV vector. * @adata_dma_list: The OCS DMA list mapping input A-data. * @adata_size: The amount of data mapped by @adata_dma_list. * @in_tag: Input tag. * @tag_size: The size (in bytes) of @in_tag. * * Note: for encrypt the tag is appended to the ciphertext (in the memory * mapped by @dst_dma_list). * * Return: 0 on success, negative error code otherwise. */ int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, enum ocs_cipher cipher, enum ocs_instruction instruction, dma_addr_t dst_dma_list, dma_addr_t src_dma_list, u32 src_size, u8 *iv, dma_addr_t adata_dma_list, u32 adata_size, u8 *in_tag, u32 tag_size) { … } /** * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. * @aes_dev: The OCS AES device the list will be created for. * @sg: The SG list OCS DMA linked list will be created from. When * passed to this function, @sg must have been already mapped * with dma_map_sg(). * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the * value returned by dma_map_sg() when @sg was mapped. * @dll_desc: The OCS DMA dma_list to use to store information about the * created linked list. * @data_size: The size of the data (from the SG list) to be mapped into the * OCS DMA linked list. * @data_offset: The offset (within the SG list) of the data to be mapped. * * Return: 0 on success, negative error code otherwise. */ int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, struct scatterlist *sg, int sg_dma_count, struct ocs_dll_desc *dll_desc, size_t data_size, size_t data_offset) { … }
Codebase Browser
linux