linux/drivers/md/dm-verity-fec.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2015 Google, Inc.
 *
 * Author: Sami Tolvanen <[email protected]>
 */

#include "dm-verity-fec.h"
#include <linux/math64.h>

#define DM_MSG_PREFIX …

/*
 * If error correction has been configured, returns true.
 */
bool verity_fec_is_enabled(struct dm_verity *v)
{ … }

/*
 * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable
 * length fields.
 */
static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io)
{ … }

/*
 * Return an interleaved offset for a byte in RS block.
 */
static inline u64 fec_interleave(struct dm_verity *v, u64 offset)
{ … }

/*
 * Decode an RS block using Reed-Solomon.
 */
static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,
			  u8 *data, u8 *fec, int neras)
{ … }

/*
 * Read error-correcting codes for the requested RS block. Returns a pointer
 * to the data block. Caller is responsible for releasing buf.
 */
static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,
			   unsigned int *offset, struct dm_buffer **buf,
			   unsigned short ioprio)
{ … }

/* Loop over each preallocated buffer slot. */
#define fec_for_each_prealloc_buffer(__i) …

/* Loop over each extra buffer slot. */
#define fec_for_each_extra_buffer(io, __i) …

/* Loop over each allocated buffer. */
#define fec_for_each_buffer(io, __i) …

/* Loop over each RS block in each allocated buffer. */
#define fec_for_each_buffer_rs_block(io, __i, __j) …

/*
 * Return a pointer to the current RS block when called inside
 * fec_for_each_buffer_rs_block.
 */
static inline u8 *fec_buffer_rs_block(struct dm_verity *v,
				      struct dm_verity_fec_io *fio,
				      unsigned int i, unsigned int j)
{ … }

/*
 * Return an index to the current RS block when called inside
 * fec_for_each_buffer_rs_block.
 */
static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j)
{ … }

/*
 * Decode all RS blocks from buffers and copy corrected bytes into fio->output
 * starting from block_offset.
 */
static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io,
			   struct dm_verity_fec_io *fio, u64 rsb, int byte_index,
			   unsigned int block_offset, int neras)
{ … }

/*
 * Locate data block erasures using verity hashes.
 */
static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
			  u8 *want_digest, u8 *data)
{ … }

/*
 * Read data blocks that are part of the RS block and deinterleave as much as
 * fits into buffers. Check for erasure locations if @neras is non-NULL.
 */
static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
			 u64 rsb, u64 target, unsigned int block_offset,
			 int *neras)
{ … }

/*
 * Allocate RS control structure and FEC buffers from preallocated mempools,
 * and attempt to allocate as many extra buffers as available.
 */
static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
{ … }

/*
 * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are
 * zeroed before deinterleaving.
 */
static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
{ … }

/*
 * Decode all RS blocks in a single data block and return the target block
 * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
 * hashes to locate erasures.
 */
static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
			  struct dm_verity_fec_io *fio, u64 rsb, u64 offset,
			  bool use_erasures)
{ … }

/* Correct errors in a block. Copies corrected block to dest. */
int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
		      enum verity_block_type type, sector_t block, u8 *dest)
{ … }

/*
 * Clean up per-bio data.
 */
void verity_fec_finish_io(struct dm_verity_io *io)
{ … }

/*
 * Initialize per-bio data.
 */
void verity_fec_init_io(struct dm_verity_io *io)
{ … }

/*
 * Append feature arguments and values to the status table.
 */
unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz,
				 char *result, unsigned int maxlen)
{ … }

void verity_fec_dtr(struct dm_verity *v)
{ … }

static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
{ … }

static void fec_rs_free(void *element, void *pool_data)
{ … }

bool verity_is_fec_opt_arg(const char *arg_name)
{ … }

int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
			      unsigned int *argc, const char *arg_name)
{ … }

/*
 * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
 */
int verity_fec_ctr_alloc(struct dm_verity *v)
{ … }

/*
 * Validate arguments and preallocate memory. Must be called after arguments
 * have been parsed using verity_fec_parse_opt_args.
 */
int verity_fec_ctr(struct dm_verity *v)
{ … }