// SPDX-License-Identifier: GPL-2.0 /* * Cryptographic API. * * Support for ATMEL AES HW acceleration. * * Copyright (c) 2012 Eukréa Electromatique - ATMEL * Author: Nicolas Royer <[email protected]> * * Some ideas are from omap-aes.c driver. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/err.h> #include <linux/clk.h> #include <linux/io.h> #include <linux/hw_random.h> #include <linux/platform_device.h> #include <linux/device.h> #include <linux/dmaengine.h> #include <linux/init.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> #include <linux/mod_devicetable.h> #include <linux/delay.h> #include <linux/crypto.h> #include <crypto/scatterwalk.h> #include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/gcm.h> #include <crypto/xts.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include "atmel-aes-regs.h" #include "atmel-authenc.h" #define ATMEL_AES_PRIORITY … #define ATMEL_AES_BUFFER_ORDER … #define ATMEL_AES_BUFFER_SIZE … #define SIZE_IN_WORDS(x) … /* AES flags */ /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ #define AES_FLAGS_ENCRYPT … #define AES_FLAGS_GTAGEN … #define AES_FLAGS_OPMODE_MASK … #define AES_FLAGS_ECB … #define AES_FLAGS_CBC … #define AES_FLAGS_CTR … #define AES_FLAGS_GCM … #define AES_FLAGS_XTS … #define AES_FLAGS_MODE_MASK … #define AES_FLAGS_BUSY … #define AES_FLAGS_DUMP_REG … #define AES_FLAGS_OWN_SHA … #define AES_FLAGS_PERSISTENT … #define ATMEL_AES_QUEUE_LENGTH … #define ATMEL_AES_DMA_THRESHOLD … struct atmel_aes_caps { … }; struct atmel_aes_dev; atmel_aes_fn_t; struct atmel_aes_base_ctx { … }; struct atmel_aes_ctx { … }; struct atmel_aes_ctr_ctx { … }; struct atmel_aes_gcm_ctx { … }; struct atmel_aes_xts_ctx { … }; #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) struct atmel_aes_authenc_ctx { … }; #endif struct atmel_aes_reqctx { … }; #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) struct atmel_aes_authenc_reqctx { … }; #endif struct atmel_aes_dma { … }; struct atmel_aes_dev { … }; struct atmel_aes_drv { … }; static struct atmel_aes_drv atmel_aes = …; #ifdef VERBOSE_DEBUG static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) { switch (offset) { case AES_CR: return "CR"; case AES_MR: return "MR"; case AES_ISR: return "ISR"; case AES_IMR: return "IMR"; case AES_IER: return "IER"; case AES_IDR: return "IDR"; case AES_KEYWR(0): case AES_KEYWR(1): case AES_KEYWR(2): case AES_KEYWR(3): case AES_KEYWR(4): case AES_KEYWR(5): case AES_KEYWR(6): case AES_KEYWR(7): snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); break; case AES_IDATAR(0): case AES_IDATAR(1): case AES_IDATAR(2): case AES_IDATAR(3): snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); break; case AES_ODATAR(0): case AES_ODATAR(1): case AES_ODATAR(2): case AES_ODATAR(3): snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); break; case AES_IVR(0): case AES_IVR(1): case AES_IVR(2): case AES_IVR(3): snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); break; case AES_AADLENR: return "AADLENR"; case AES_CLENR: return "CLENR"; case AES_GHASHR(0): case AES_GHASHR(1): case AES_GHASHR(2): case AES_GHASHR(3): snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); break; case AES_TAGR(0): case AES_TAGR(1): case AES_TAGR(2): case AES_TAGR(3): snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); break; case AES_CTRR: return "CTRR"; case AES_GCMHR(0): case AES_GCMHR(1): case AES_GCMHR(2): case AES_GCMHR(3): snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); break; case AES_EMR: return "EMR"; case AES_TWR(0): case AES_TWR(1): case AES_TWR(2): case AES_TWR(3): snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); break; case AES_ALPHAR(0): case AES_ALPHAR(1): case AES_ALPHAR(2): case AES_ALPHAR(3): snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); break; default: snprintf(tmp, sz, "0x%02x", offset); break; } return tmp; } #endif /* VERBOSE_DEBUG */ /* Shared functions */ static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) { … } static inline void atmel_aes_write(struct atmel_aes_dev *dd, u32 offset, u32 value) { … } static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, u32 *value, int count) { … } static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, const u32 *value, int count) { … } static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, void *value) { … } static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, const void *value) { … } static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, atmel_aes_fn_t resume) { … } static inline size_t atmel_aes_padlen(size_t len, size_t block_size) { … } static struct atmel_aes_dev *atmel_aes_dev_alloc(struct atmel_aes_base_ctx *ctx) { … } static int atmel_aes_hw_init(struct atmel_aes_dev *dd) { … } static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) { … } static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) { … } static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, const struct atmel_aes_reqctx *rctx) { … } static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) { … } #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); #endif static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) { … } static inline struct atmel_aes_ctr_ctx * atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) { … } static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd) { … } static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) { … } static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, const __be32 *iv, const u32 *key, int keylen) { … } static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, const __be32 *iv) { … } /* CPU transfer */ static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) { … } static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, struct scatterlist *src, struct scatterlist *dst, size_t len, atmel_aes_fn_t resume) { … } /* DMA transfer */ static void atmel_aes_dma_callback(void *data); static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, struct scatterlist *sg, size_t len, struct atmel_aes_dma *dma) { … } static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) { … } static int atmel_aes_map(struct atmel_aes_dev *dd, struct scatterlist *src, struct scatterlist *dst, size_t len) { … } static void atmel_aes_unmap(struct atmel_aes_dev *dd) { … } static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, enum dma_slave_buswidth addr_width, enum dma_transfer_direction dir, u32 maxburst) { … } static int atmel_aes_dma_start(struct atmel_aes_dev *dd, struct scatterlist *src, struct scatterlist *dst, size_t len, atmel_aes_fn_t resume) { … } static void atmel_aes_dma_callback(void *data) { … } static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, struct crypto_async_request *new_areq) { … } /* AES async block ciphers */ static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) { … } static int atmel_aes_start(struct atmel_aes_dev *dd) { … } static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) { … } static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) { … } static int atmel_aes_xts_fallback(struct skcipher_request *req, bool enc) { … } static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) { … } static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static int atmel_aes_ecb_encrypt(struct skcipher_request *req) { … } static int atmel_aes_ecb_decrypt(struct skcipher_request *req) { … } static int atmel_aes_cbc_encrypt(struct skcipher_request *req) { … } static int atmel_aes_cbc_decrypt(struct skcipher_request *req) { … } static int atmel_aes_ctr_encrypt(struct skcipher_request *req) { … } static int atmel_aes_ctr_decrypt(struct skcipher_request *req) { … } static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) { … } static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) { … } static struct skcipher_alg aes_algs[] = …; /* gcm aead functions */ static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, const u32 *data, size_t datalen, const __be32 *ghash_in, __be32 *ghash_out, atmel_aes_fn_t resume); static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); static inline struct atmel_aes_gcm_ctx * atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) { … } static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, const u32 *data, size_t datalen, const __be32 *ghash_in, __be32 *ghash_out, atmel_aes_fn_t resume) { … } static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) { … } static int atmel_aes_gcm_crypt(struct aead_request *req, unsigned long mode) { … } static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { … } static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { … } static int atmel_aes_gcm_encrypt(struct aead_request *req) { … } static int atmel_aes_gcm_decrypt(struct aead_request *req) { … } static int atmel_aes_gcm_init(struct crypto_aead *tfm) { … } static struct aead_alg aes_gcm_alg = …; /* xts functions */ static inline struct atmel_aes_xts_ctx * atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) { … } static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); static int atmel_aes_xts_start(struct atmel_aes_dev *dd) { … } static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) { … } static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { … } static int atmel_aes_xts_encrypt(struct skcipher_request *req) { … } static int atmel_aes_xts_decrypt(struct skcipher_request *req) { … } static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) { … } static void atmel_aes_xts_exit_tfm(struct crypto_skcipher *tfm) { … } static struct skcipher_alg aes_xts_alg = …; #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) /* authenc aead functions */ static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, bool is_async); static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, bool is_async); static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, bool is_async); static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) { … } static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) { … } static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, bool is_async) { … } static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, bool is_async) { … } static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) { … } static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, bool is_async) { … } static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { … } static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, unsigned long auth_mode) { … } static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) { … } static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) { … } static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) { … } static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) { … } static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) { … } static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) { … } static int atmel_aes_authenc_crypt(struct aead_request *req, unsigned long mode) { … } static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) { … } static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) { … } static struct aead_alg aes_authenc_algs[] = …; #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ /* Probe functions */ static int atmel_aes_buff_init(struct atmel_aes_dev *dd) { … } static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) { … } static int atmel_aes_dma_init(struct atmel_aes_dev *dd) { … } static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) { … } static void atmel_aes_queue_task(unsigned long data) { … } static void atmel_aes_done_task(unsigned long data) { … } static irqreturn_t atmel_aes_irq(int irq, void *dev_id) { … } static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) { … } static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) { … } static int atmel_aes_register_algs(struct atmel_aes_dev *dd) { … } static void atmel_aes_get_cap(struct atmel_aes_dev *dd) { … } static const struct of_device_id atmel_aes_dt_ids[] = …; MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); static int atmel_aes_probe(struct platform_device *pdev) { … } static void atmel_aes_remove(struct platform_device *pdev) { … } static struct platform_driver atmel_aes_driver = …; module_platform_driver(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …; MODULE_AUTHOR(…) …;
Codebase Browser
linux