linux/arch/x86/crypto/chacha_glue.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * x64 SIMD accelerated ChaCha and XChaCha stream ciphers,
 * including ChaCha20 (RFC7539)
 *
 * Copyright (C) 2015 Martin Willi
 */

#include <crypto/algapi.h>
#include <crypto/internal/chacha.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sizes.h>
#include <asm/simd.h>

asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
				       unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
					unsigned int len, int nrounds);
asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);

asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
				       unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
				       unsigned int len, int nrounds);
asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
				       unsigned int len, int nrounds);

asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
					   unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
					   unsigned int len, int nrounds);
asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
					   unsigned int len, int nrounds);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl);

static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks)
{ … }

static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
			  unsigned int bytes, int nrounds)
{ … }

void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
{ … }
EXPORT_SYMBOL(…);

void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
{ … }
EXPORT_SYMBOL(…);

void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
		       int nrounds)
{ … }
EXPORT_SYMBOL(…);

static int chacha_simd_stream_xor(struct skcipher_request *req,
				  const struct chacha_ctx *ctx, const u8 *iv)
{ … }

static int chacha_simd(struct skcipher_request *req)
{ … }

static int xchacha_simd(struct skcipher_request *req)
{ … }

static struct skcipher_alg algs[] = …;

static int __init chacha_simd_mod_init(void)
{ … }

static void __exit chacha_simd_mod_fini(void)
{ … }

module_init(…) …;
module_exit(chacha_simd_mod_fini);

MODULE_LICENSE(…) …;
MODULE_AUTHOR(…) …;
MODULE_DESCRIPTION(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;