// SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019 HiSilicon Limited. */ #include <crypto/aes.h> #include <crypto/aead.h> #include <crypto/algapi.h> #include <crypto/authenc.h> #include <crypto/des.h> #include <crypto/hash.h> #include <crypto/internal/aead.h> #include <crypto/internal/des.h> #include <crypto/sha1.h> #include <crypto/sha2.h> #include <crypto/skcipher.h> #include <crypto/xts.h> #include <linux/crypto.h> #include <linux/dma-mapping.h> #include <linux/idr.h> #include "sec.h" #include "sec_crypto.h" #define SEC_PRIORITY … #define SEC_XTS_MIN_KEY_SIZE … #define SEC_XTS_MID_KEY_SIZE … #define SEC_XTS_MAX_KEY_SIZE … #define SEC_DES3_2KEY_SIZE … #define SEC_DES3_3KEY_SIZE … /* SEC sqe(bd) bit operational relative MACRO */ #define SEC_DE_OFFSET … #define SEC_CIPHER_OFFSET … #define SEC_SCENE_OFFSET … #define SEC_DST_SGL_OFFSET … #define SEC_SRC_SGL_OFFSET … #define SEC_CKEY_OFFSET … #define SEC_CMODE_OFFSET … #define SEC_AKEY_OFFSET … #define SEC_AEAD_ALG_OFFSET … #define SEC_AUTH_OFFSET … #define SEC_DE_OFFSET_V3 … #define SEC_SCENE_OFFSET_V3 … #define SEC_CKEY_OFFSET_V3 … #define SEC_CTR_CNT_OFFSET … #define SEC_CTR_CNT_ROLLOVER … #define SEC_SRC_SGL_OFFSET_V3 … #define SEC_DST_SGL_OFFSET_V3 … #define SEC_CALG_OFFSET_V3 … #define SEC_AKEY_OFFSET_V3 … #define SEC_MAC_OFFSET_V3 … #define SEC_AUTH_ALG_OFFSET_V3 … #define SEC_CIPHER_AUTH_V3 … #define SEC_AUTH_CIPHER_V3 … #define SEC_FLAG_OFFSET … #define SEC_FLAG_MASK … #define SEC_TYPE_MASK … #define SEC_DONE_MASK … #define SEC_ICV_MASK … #define SEC_SQE_LEN_RATE_MASK … #define SEC_TOTAL_IV_SZ(depth) … #define SEC_SGL_SGE_NR … #define SEC_CIPHER_AUTH … #define SEC_AUTH_CIPHER … #define SEC_MAX_MAC_LEN … #define SEC_MAX_AAD_LEN … #define SEC_MAX_CCM_AAD_LEN … #define SEC_TOTAL_MAC_SZ(depth) … #define SEC_PBUF_SZ … #define SEC_PBUF_IV_OFFSET … #define SEC_PBUF_MAC_OFFSET … #define SEC_PBUF_PKG … #define SEC_PBUF_NUM … #define SEC_PBUF_PAGE_NUM(depth) … #define SEC_PBUF_LEFT_SZ(depth) … #define SEC_TOTAL_PBUF_SZ(depth) … #define SEC_SQE_LEN_RATE … #define SEC_SQE_CFLAG … #define SEC_SQE_AEAD_FLAG … #define SEC_SQE_DONE … #define SEC_ICV_ERR … #define MIN_MAC_LEN … #define MAC_LEN_MASK … #define MAX_INPUT_DATA_LEN … #define BITS_MASK … #define BYTE_BITS … #define SEC_XTS_NAME_SZ … #define IV_CM_CAL_NUM … #define IV_CL_MASK … #define IV_CL_MIN … #define IV_CL_MID … #define IV_CL_MAX … #define IV_FLAGS_OFFSET … #define IV_CM_OFFSET … #define IV_LAST_BYTE1 … #define IV_LAST_BYTE2 … #define IV_LAST_BYTE_MASK … #define IV_CTR_INIT … #define IV_BYTE_OFFSET … static DEFINE_MUTEX(sec_algs_lock); static unsigned int sec_available_devs; struct sec_skcipher { … }; struct sec_aead { … }; /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { … } static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) { … } static void sec_free_req_id(struct sec_req *req) { … } static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) { … } static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) { … } static int sec_cb_status_check(struct sec_req *req, struct bd_status *status) { … } static void sec_req_cb(struct hisi_qp *qp, void *resp) { … } static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) { … } /* Get DMA memory resources */ static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) { … } static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) { … } static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) { … } static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) { … } static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) { … } static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) { … } static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) { … } /* * To improve performance, pbuffer is used for * small packets (< 512Bytes) as IOMMU translation using. */ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) { … } static int sec_alg_resource_alloc(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static void sec_alg_resource_free(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) { … } static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static int sec_ctx_base_init(struct sec_ctx *ctx) { … } static void sec_ctx_base_uninit(struct sec_ctx *ctx) { … } static int sec_cipher_init(struct sec_ctx *ctx) { … } static void sec_cipher_uninit(struct sec_ctx *ctx) { … } static int sec_auth_init(struct sec_ctx *ctx) { … } static void sec_auth_uninit(struct sec_ctx *ctx) { … } static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) { … } static int sec_skcipher_init(struct crypto_skcipher *tfm) { … } static void sec_skcipher_uninit(struct crypto_skcipher *tfm) { … } static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen) { … } static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, const u32 keylen, const enum sec_cmode c_mode) { … } static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen, const enum sec_calg c_alg, const enum sec_cmode c_mode) { … } #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) … GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src) { … } static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *dst) { … } static int sec_aead_mac_init(struct sec_aead_req *req) { … } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { … } static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src, struct scatterlist *dst) { … } static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, struct crypto_authenc_keys *keys) { … } static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_authenc_keys *keys) { … } static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) { … } static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { … } static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, const u32 keylen, const enum sec_hash_alg a_alg, const enum sec_calg c_alg, const enum sec_mac_len mac_len, const enum sec_cmode c_mode) { … } #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) … GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) { … } /* increment counter (128-bit int) */ static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) { … } static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) { … } static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { … } static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, int err) { … } static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, struct sec_req *req, struct sec_sqe *sec_sqe) { … } static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, struct sec_req *req, struct sec_sqe3 *sqe3) { … } static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, struct sec_req *req, struct sec_sqe *sec_sqe) { … } static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, struct sec_req *req, struct sec_sqe3 *sqe3) { … } static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) { … } static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) { … } static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) { … } static int sec_process(struct sec_ctx *ctx, struct sec_req *req) { … } static const struct sec_req_op sec_skcipher_req_ops = …; static const struct sec_req_op sec_aead_req_ops = …; static const struct sec_req_op sec_skcipher_req_ops_v3 = …; static const struct sec_req_op sec_aead_req_ops_v3 = …; static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) { … } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) { … } static int sec_aead_init(struct crypto_aead *tfm) { … } static void sec_aead_exit(struct crypto_aead *tfm) { … } static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) { … } static void sec_aead_ctx_exit(struct crypto_aead *tfm) { … } static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) { … } static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) { … } static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) { … } static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) { … } static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) { … } static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, struct sec_req *sreq) { … } static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { … } static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, struct skcipher_request *sreq, bool encrypt) { … } static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) { … } static int sec_skcipher_encrypt(struct skcipher_request *sk_req) { … } static int sec_skcipher_decrypt(struct skcipher_request *sk_req) { … } #define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \ sec_min_key_size, sec_max_key_size, blk_size, iv_size) … static struct sec_skcipher sec_skciphers[] = …; static int aead_iv_demension_check(struct aead_request *aead_req) { … } static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) { … } static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { … } static int sec_aead_soft_crypto(struct sec_ctx *ctx, struct aead_request *aead_req, bool encrypt) { … } static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) { … } static int sec_aead_encrypt(struct aead_request *a_req) { … } static int sec_aead_decrypt(struct aead_request *a_req) { … } #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ ctx_exit, blk_size, iv_size, max_authsize) … static struct sec_aead sec_aeads[] = …; static void sec_unregister_skcipher(u64 alg_mask, int end) { … } static int sec_register_skcipher(u64 alg_mask) { … } static void sec_unregister_aead(u64 alg_mask, int end) { … } static int sec_register_aead(u64 alg_mask) { … } int sec_register_to_crypto(struct hisi_qm *qm) { … } void sec_unregister_from_crypto(struct hisi_qm *qm) { … }
Codebase Browser
linux