linux/arch/x86/crypto/poly1305_glue.c

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <[email protected]>. All Rights Reserved.
 */

#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/poly1305.h>
#include <crypto/internal/simd.h>
#include <linux/crypto.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sizes.h>
#include <asm/cpu_device_id.h>
#include <asm/simd.h>

asmlinkage void poly1305_init_x86_64(void *ctx,
				     const u8 key[POLY1305_BLOCK_SIZE]);
asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
				       const size_t len, const u32 padbit);
asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
				     const u32 nonce[4]);
asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
				  const u32 nonce[4]);
asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
				    const u32 padbit);
asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
				     const u32 padbit);
asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
				       const size_t len, const u32 padbit);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);

struct poly1305_arch_internal { … };

/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit
 * the unfortunate situation of using AVX and then having to go back to scalar
 * -- because the user is silly and has called the update function from two
 * separate contexts -- then we need to convert back to the original base before
 * proceeding. It is possible to reason that the initial reduction below is
 * sufficient given the implementation invariants. However, for an avoidance of
 * doubt and because this is not performance critical, we do the full reduction
 * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py
 */
static void convert_to_base2_64(void *ctx)
{ … }

static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE])
{ … }

static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len,
				 const u32 padbit)
{ … }

static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
			       const u32 nonce[4])
{ … }

void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
{ … }
EXPORT_SYMBOL(…);

static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
					       const u8 *inp, unsigned int len)
{ … }

void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
			  unsigned int srclen)
{ … }
EXPORT_SYMBOL(…);

void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
{ … }
EXPORT_SYMBOL(…);

static int crypto_poly1305_init(struct shash_desc *desc)
{ … }

static int crypto_poly1305_update(struct shash_desc *desc,
				  const u8 *src, unsigned int srclen)
{ … }

static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
{ … }

static struct shash_alg alg = …;

static int __init poly1305_simd_mod_init(void)
{ … }

static void __exit poly1305_simd_mod_exit(void)
{ … }

module_init(…) …;
module_exit(poly1305_simd_mod_exit);

MODULE_LICENSE(…) …;
MODULE_AUTHOR(…) …;
MODULE_DESCRIPTION(…) …;
MODULE_ALIAS_CRYPTO(…) …;
MODULE_ALIAS_CRYPTO(…) …;