blk-crypto-fallback.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2019 Google LLC
 */

/*
 * Refer to Documentation/block/inline-encryption.rst for detailed explanation.
 */

#define pr_fmt(fmt) …

#include <crypto/skcipher.h>
#include <linux/blk-crypto.h>
#include <linux/blk-crypto-profile.h>
#include <linux/blkdev.h>
#include <linux/crypto.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/scatterlist.h>

#include "blk-cgroup.h"
#include "blk-crypto-internal.h"

static unsigned int num_prealloc_bounce_pg = …;
module_param(num_prealloc_bounce_pg, uint, 0);
MODULE_PARM_DESC(…) …;

static unsigned int blk_crypto_num_keyslots = …;
module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0);
MODULE_PARM_DESC(…) …;

static unsigned int num_prealloc_fallback_crypt_ctxs = …;
module_param(num_prealloc_fallback_crypt_ctxs, uint, 0);
MODULE_PARM_DESC(…) …;

struct bio_fallback_crypt_ctx { … };

static struct kmem_cache *bio_fallback_crypt_ctx_cache;
static mempool_t *bio_fallback_crypt_ctx_pool;

/*
 * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
 * all of a mode's tfms when that mode starts being used. Since each mode may
 * need all the keyslots at some point, each mode needs its own tfm for each
 * keyslot; thus, a keyslot may contain tfms for multiple modes.  However, to
 * match the behavior of real inline encryption hardware (which only supports a
 * single encryption context per keyslot), we only allow one tfm per keyslot to
 * be used at a time - the rest of the unused tfms have their keys cleared.
 */
static DEFINE_MUTEX(tfms_init_lock);
static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX];

static struct blk_crypto_fallback_keyslot { … } *blk_crypto_keyslots;

static struct blk_crypto_profile *blk_crypto_fallback_profile;
static struct workqueue_struct *blk_crypto_wq;
static mempool_t *blk_crypto_bounce_page_pool;
static struct bio_set crypto_bio_split;

/*
 * This is the key we set when evicting a keyslot. This *should* be the all 0's
 * key, but AES-XTS rejects that key, so we use some random bytes instead.
 */
static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];

static void blk_crypto_fallback_evict_keyslot(unsigned int slot)
{ … }

static int
blk_crypto_fallback_keyslot_program(struct blk_crypto_profile *profile,
				    const struct blk_crypto_key *key,
				    unsigned int slot)
{ … }

static int blk_crypto_fallback_keyslot_evict(struct blk_crypto_profile *profile,
					     const struct blk_crypto_key *key,
					     unsigned int slot)
{ … }

static const struct blk_crypto_ll_ops blk_crypto_fallback_ll_ops = …;

static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio)
{ … }

static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src)
{ … }

static bool
blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot,
				     struct skcipher_request **ciph_req_ret,
				     struct crypto_wait *wait)
{ … }

static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr)
{ … }

blk_crypto_iv;

static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
				 union blk_crypto_iv *iv)
{ … }

/*
 * The crypto API fallback's encryption routine.
 * Allocate a bounce bio for encryption, encrypt the input bio using crypto API,
 * and replace *bio_ptr with the bounce bio. May split input bio if it's too
 * large. Returns true on success. Returns false and sets bio->bi_status on
 * error.
 */
static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr)
{ … }

/*
 * The crypto API fallback's main decryption routine.
 * Decrypts input bio in place, and calls bio_endio on the bio.
 */
static void blk_crypto_fallback_decrypt_bio(struct work_struct *work)
{ … }

/**
 * blk_crypto_fallback_decrypt_endio - queue bio for fallback decryption
 *
 * @bio: the bio to queue
 *
 * Restore bi_private and bi_end_io, and queue the bio for decryption into a
 * workqueue, since this function will be called from an atomic context.
 */
static void blk_crypto_fallback_decrypt_endio(struct bio *bio)
{ … }

/**
 * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption
 *
 * @bio_ptr: pointer to the bio to prepare
 *
 * If bio is doing a WRITE operation, this splits the bio into two parts if it's
 * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a
 * bounce bio for the first part, encrypts it, and updates bio_ptr to point to
 * the bounce bio.
 *
 * For a READ operation, we mark the bio for decryption by using bi_private and
 * bi_end_io.
 *
 * In either case, this function will make the bio look like a regular bio (i.e.
 * as if no encryption context was ever specified) for the purposes of the rest
 * of the stack except for blk-integrity (blk-integrity and blk-crypto are not
 * currently supported together).
 *
 * Return: true on success. Sets bio->bi_status and returns false on error.
 */
bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr)
{ … }

int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key)
{ … }

static bool blk_crypto_fallback_inited;
static int blk_crypto_fallback_init(void)
{ … }

/*
 * Prepare blk-crypto-fallback for the specified crypto mode.
 * Returns -ENOPKG if the needed crypto API support is missing.
 */
int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num)
{ … }
linux/block/blk-crypto-fallback.c
