// SPDX-License-Identifier: GPL-2.0 /* * K3 SA2UL crypto accelerator driver * * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com * * Authors: Keerthy * Vitaly Andrianov * Tero Kristo */ #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/dmapool.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <crypto/aes.h> #include <crypto/authenc.h> #include <crypto/des.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <crypto/sha1.h> #include <crypto/sha2.h> #include "sa2ul.h" /* Byte offset for key in encryption security context */ #define SC_ENC_KEY_OFFSET … /* Byte offset for Aux-1 in encryption security context */ #define SC_ENC_AUX1_OFFSET … #define SA_CMDL_UPD_ENC … #define SA_CMDL_UPD_AUTH … #define SA_CMDL_UPD_ENC_IV … #define SA_CMDL_UPD_AUTH_IV … #define SA_CMDL_UPD_AUX_KEY … #define SA_AUTH_SUBKEY_LEN … #define SA_CMDL_PAYLOAD_LENGTH_MASK … #define SA_CMDL_SOP_BYPASS_LEN_MASK … #define MODE_CONTROL_BYTES … #define SA_HASH_PROCESSING … #define SA_CRYPTO_PROCESSING … #define SA_UPLOAD_HASH_TO_TLR … #define SA_SW0_FLAGS_MASK … #define SA_SW0_CMDL_INFO_MASK … #define SA_SW0_CMDL_PRESENT … #define SA_SW0_ENG_ID_MASK … #define SA_SW0_DEST_INFO_PRESENT … #define SA_SW2_EGRESS_LENGTH … #define SA_BASIC_HASH … #define SHA256_DIGEST_WORDS … /* Make 32-bit word from 4 bytes */ #define SA_MK_U32(b0, b1, b2, b3) … /* size of SCCTL structure in bytes */ #define SA_SCCTL_SZ … /* Max Authentication tag size */ #define SA_MAX_AUTH_TAG_SZ … enum sa_algo_id { … }; struct sa_match_data { … }; static struct device *sa_k3_dev; /** * struct sa_cmdl_cfg - Command label configuration descriptor * @aalg: authentication algorithm ID * @enc_eng_id: Encryption Engine ID supported by the SA hardware * @auth_eng_id: Authentication Engine ID * @iv_size: Initialization Vector size * @akey: Authentication key * @akey_len: Authentication key length * @enc: True, if this is an encode request */ struct sa_cmdl_cfg { … }; /** * struct algo_data - Crypto algorithm specific data * @enc_eng: Encryption engine info structure * @auth_eng: Authentication engine info structure * @auth_ctrl: Authentication control word * @hash_size: Size of digest * @iv_idx: iv index in psdata * @iv_out_size: iv out size * @ealg_id: Encryption Algorithm ID * @aalg_id: Authentication algorithm ID * @mci_enc: Mode Control Instruction for Encryption algorithm * @mci_dec: Mode Control Instruction for Decryption * @inv_key: Whether the encryption algorithm demands key inversion * @ctx: Pointer to the algorithm context * @keyed_mac: Whether the authentication algorithm has key * @prep_iopad: Function pointer to generate intermediate ipad/opad */ struct algo_data { … }; /** * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms * @type: Type of the crypto algorithm. * @alg: Union of crypto algorithm definitions. * @registered: Flag indicating if the crypto algorithm is already registered */ struct sa_alg_tmpl { … }; /** * struct sa_mapped_sg: scatterlist information for tx and rx * @mapped: Set to true if the @sgt is mapped * @dir: mapping direction used for @sgt * @split_sg: Set if the sg is split and needs to be freed up * @static_sg: Static scatterlist entry for overriding data * @sgt: scatterlist table for DMA API use */ struct sa_mapped_sg { … }; /** * struct sa_rx_data: RX Packet miscellaneous data place holder * @req: crypto request data pointer * @ddev: pointer to the DMA device * @tx_in: dma_async_tx_descriptor pointer for rx channel * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping * @enc: Flag indicating either encryption or decryption * @enc_iv_size: Initialisation vector size * @iv_idx: Initialisation vector index */ struct sa_rx_data { … }; /** * struct sa_req: SA request definition * @dev: device for the request * @size: total data to the xmitted via DMA * @enc_offset: offset of cipher data * @enc_size: data to be passed to cipher engine * @enc_iv: cipher IV * @auth_offset: offset of the authentication data * @auth_size: size of the authentication data * @auth_iv: authentication IV * @type: algorithm type for the request * @cmdl: command label pointer * @base: pointer to the base request * @ctx: pointer to the algorithm context data * @enc: true if this is an encode request * @src: source data * @dst: destination data * @callback: DMA callback for the request * @mdata_size: metadata size passed to DMA */ struct sa_req { … }; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For CBC (Cipher Block Chaining) mode for encryption */ static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = …; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For CBC (Cipher Block Chaining) mode for decryption */ static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = …; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For CBC (Cipher Block Chaining) mode for encryption */ static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = …; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For CBC (Cipher Block Chaining) mode for decryption */ static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = …; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For ECB (Electronic Code Book) mode for encryption */ static u8 mci_ecb_enc_array[3][27] = …; /* * Mode Control Instructions for various Key lengths 128, 192, 256 * For ECB (Electronic Code Book) mode for decryption */ static u8 mci_ecb_dec_array[3][27] = …; /* * Mode Control Instructions for DES algorithm * For CBC (Cipher Block Chaining) mode and ECB mode * encryption and for decryption respectively */ static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = …; static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = …; static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = …; static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = …; /* * Perform 16 byte or 128 bit swizzling * The SA2UL Expects the security context to * be in little Endian and the bus width is 128 bits or 16 bytes * Hence swap 16 bytes at a time from higher to lower address */ static void sa_swiz_128(u8 *in, u16 len) { … } /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) { … } static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) { … } static void sa_export_shash(void *state, struct shash_desc *hash, int digest_size, __be32 *out) { … } static void sa_prepare_iopads(struct algo_data *data, const u8 *key, u16 key_sz, __be32 *ipad, __be32 *opad) { … } /* Derive the inverse key used in AES-CBC decryption operation */ static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) { … } /* Set Security context for the encryption engine */ static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, u8 enc, u8 *sc_buf) { … } /* Set Security context for the authentication engine */ static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, u8 *sc_buf) { … } static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) { … } /* Format general command label */ static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, struct sa_cmdl_upd_info *upd_info) { … } /* Update Command label */ static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, struct sa_cmdl_upd_info *upd_info) { … } /* Format SWINFO words to be sent to SA */ static void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, u8 cmdl_present, u8 cmdl_offset, u8 flags, u8 hash_size, u32 *swinfo) { … } /* Dump the security context */ static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) { … } static int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data, const u8 *enc_key, u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz, struct algo_data *ad, u8 enc, u32 *swinfo) { … } /* Free the per direction context memory */ static void sa_free_ctx_info(struct sa_ctx_info *ctx, struct sa_crypto_data *data) { … } static int sa_init_ctx_info(struct sa_ctx_info *ctx, struct sa_crypto_data *data) { … } static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) { … } static int sa_cipher_cra_init(struct crypto_skcipher *tfm) { … } static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen, struct algo_data *ad) { … } static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static void sa_sync_from_device(struct sa_rx_data *rxd) { … } static void sa_free_sa_rx_data(struct sa_rx_data *rxd) { … } static void sa_aes_dma_in_callback(void *data) { … } static void sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) { … } static int sa_run(struct sa_req *req) { … } static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) { … } static int sa_encrypt(struct skcipher_request *req) { … } static int sa_decrypt(struct skcipher_request *req) { … } static void sa_sha_dma_in_callback(void *data) { … } static int zero_message_process(struct ahash_request *req) { … } static int sa_sha_run(struct ahash_request *req) { … } static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) { … } static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) { … } static int sa_sha_digest(struct ahash_request *req) { … } static int sa_sha_init(struct ahash_request *req) { … } static int sa_sha_update(struct ahash_request *req) { … } static int sa_sha_final(struct ahash_request *req) { … } static int sa_sha_finup(struct ahash_request *req) { … } static int sa_sha_import(struct ahash_request *req, const void *in) { … } static int sa_sha_export(struct ahash_request *req, void *out) { … } static int sa_sha1_cra_init(struct crypto_tfm *tfm) { … } static int sa_sha256_cra_init(struct crypto_tfm *tfm) { … } static int sa_sha512_cra_init(struct crypto_tfm *tfm) { … } static void sa_sha_cra_exit(struct crypto_tfm *tfm) { … } static void sa_aead_dma_in_callback(void *data) { … } static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, const char *fallback) { … } static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) { … } static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) { … } static void sa_exit_tfm_aead(struct crypto_aead *tfm) { … } /* AEAD algorithm configuration interface function */ static int sa_aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen, struct algo_data *ad) { … } static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { … } static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { … } static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { … } static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) { … } /* AEAD algorithm encrypt interface function */ static int sa_aead_encrypt(struct aead_request *req) { … } /* AEAD algorithm decrypt interface function */ static int sa_aead_decrypt(struct aead_request *req) { … } static struct sa_alg_tmpl sa_algs[] = …; /* Register the algorithms in crypto framework */ static void sa_register_algos(struct sa_crypto_data *dev_data) { … } /* Unregister the algorithms in crypto framework */ static void sa_unregister_algos(const struct device *dev) { … } static int sa_init_mem(struct sa_crypto_data *dev_data) { … } static int sa_dma_init(struct sa_crypto_data *dd) { … } static int sa_link_child(struct device *dev, void *data) { … } static struct sa_match_data am654_match_data = …; static struct sa_match_data am64_match_data = …; static const struct of_device_id of_match[] = …; MODULE_DEVICE_TABLE(of, of_match); static int sa_ul_probe(struct platform_device *pdev) { … } static void sa_ul_remove(struct platform_device *pdev) { … } static struct platform_driver sa_ul_driver = …; module_platform_driver(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux