chromium/third_party/xnnpack/src/src/xnnpack/microfnptr.h

// Copyright 2022 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#pragma once

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

#include "xnnpack/common.h"
#include "xnnpack/microparams.h"


/****************** Microkernel pointers for dense inference *****************/

// CONV-HWC: direct CONVolution in HWC layout

xnn_conv_hwc_ukernel_fn;

xnn_f32_conv_hwc_ukernel_fn;

// GEMM: GEneral Matrix Multiplication without activations

xnn_gemm_ukernel_fn;

xnn_dqgemm_ukernel_fn;

xnn_dqgemm_bl_ukernel_fn;

xnn_f32_gemm_ukernel_fn;

xnn_f32_qc8w_gemm_ukernel_fn;

// GEMM: GEneral Matrix Multiplication with ReLU activation

xnn_f32_gemm_relu_ukernel_fn;

xnn_f32_qc8w_gemm_relu_ukernel_fn;

// GEneral Matrix Multiplication with post operations

xnn_f32_gemm_post_operation_ukernel_fn;

// GEMM: GEneral Matrix Multiplication with Min+Max activation

xnn_bf16_gemm_minmax_ukernel_fn;

xnn_f16_gemm_minmax_ukernel_fn;

xnn_f32_gemm_minmax_ukernel_fn;

xnn_f32_gemm_goi_minmax_ukernel_fn;

xnn_f32_qc4w_gemm_minmax_ukernel_fn;

xnn_f32_qc8w_gemm_minmax_ukernel_fn;

xnn_qs8_gemm_minmax_ukernel_fn;

xnn_qd8_f16_qc8w_gemm_ukernel_fn;

xnn_qd8_f32_qc8w_gemm_ukernel_fn;

xnn_qd8_f16_qc4w_gemm_ukernel_fn;

xnn_qd8_f16_qb4w_gemm_ukernel_fn;

xnn_qd8_f32_qc4w_gemm_ukernel_fn;

xnn_qd8_f32_qb4w_gemm_ukernel_fn;

xnn_qs8_qc8w_gemm_minmax_ukernel_fn;

xnn_qu8_gemm_minmax_ukernel_fn;

// GEMM: GEneral Matrix Multiplication with packed and quantized LHS operand.

xnn_qp8_f32_qc4w_gemm_minmax_ukernel_fn;

// GEMMINC: GEMM INCremental with Min+Max activation

xnn_f32_gemminc_minmax_ukernel_fn;

// IGEMM: Indirect GEMM without activation

xnn_dqigemm_ukernel_fn;

xnn_igemm_ukernel_fn;

xnn_f32_igemm_ukernel_fn;

// IGEMM: Indirect GEMM with ReLU activation

xnn_f32_igemm_relu_ukernel_fn;

// IGEMM: Indirect GEMM with Min+Max activation

xnn_f16_igemm_minmax_ukernel_fn;

xnn_f32_igemm_minmax_ukernel_fn;

// IGEMM: Indirect GEMM with post operations

xnn_f32_igemm_post_operation_ukernel_fn;

xnn_qd8_f16_qc8w_igemm_ukernel_fn;

xnn_qd8_f32_qc8w_igemm_ukernel_fn;

xnn_qs8_igemm_minmax_ukernel_fn;

xnn_qs8_qc8w_igemm_minmax_ukernel_fn;

xnn_qu8_igemm_minmax_ukernel_fn;

// PPMM: Pre-Packed Matrix Multiplication)

xnn_ppmm_ukernel_fn;

xnn_f16_ppmm_ukernel_fn;

xnn_f32_ppmm_minmax_ukernel_fn;

// DWCONV: DepthWise CONVolution single-pass without activation

xnn_dwconv_unipass_ukernel_fn;

xnn_f32_dwconv_unipass_ukernel_fn;

// DWCONV: DepthWise CONVolution single-pass with Min+Max activation

xnn_f16_dwconv_minmax_unipass_ukernel_fn;

xnn_f32_dwconv_minmax_unipass_ukernel_fn;

xnn_qs8_dwconv_minmax_unipass_ukernel_fn;

xnn_qs8_qc8w_dwconv_minmax_unipass_ukernel_fn;

xnn_qu8_dwconv_minmax_unipass_ukernel_fn;

// DWCONV: DepthWise CONVolution multi-pass without activation

xnn_dwconv_multipass_ukernel_fn;

xnn_f32_dwconv_multipass_ukernel_fn;

// DWCONV: DepthWise CONVolution multi-pass with Min+Max activation

xnn_f16_dwconv_minmax_multipass_ukernel_fn;

xnn_f32_dwconv_minmax_multipass_ukernel_fn;

xnn_qs8_dwconv_minmax_multipass_ukernel_fn;

xnn_qu8_dwconv_minmax_multipass_ukernel_fn;

xnn_qs8_qc8w_dwconv_minmax_multipass_ukernel_fn;

// VMULCADDC: Vector MULtiply-by-Constant, ADD-Constant

xnn_vmulcaddc_ukernel_fn;

xnn_f16_vmulcaddc_ukernel_fn;

xnn_f32_vmulcaddc_ukernel_fn;

// PRELU: Parametric RELU

xnn_prelu_ukernel_fn;

xnn_f16_prelu_ukernel_fn;

xnn_f32_prelu_ukernel_fn;

// IBILINEAR: Indirect BILINEAR interpolation

xnn_ibilinear_ukernel_fn;

xnn_f16_ibilinear_ukernel_fn;

xnn_f32_ibilinear_ukernel_fn;

xnn_s8_ibilinear_ukernel_fn;

xnn_u8_ibilinear_ukernel_fn;

// GAVGPOOL: Global AVeraGe POOLing single-pass

xnn_gavgpool_unipass_ukernel_fn;

xnn_f16_gavgpool_minmax_unipass_ukernel_fn;

xnn_f32_gavgpool_minmax_unipass_ukernel_fn;

xnn_qs8_gavgpool_minmax_unipass_ukernel_fn;

xnn_qu8_gavgpool_minmax_unipass_ukernel_fn;

// GAVGPOOL: Global AVeraGe POOLing multi-pass

xnn_gavgpool_multipass_ukernel_fn;

xnn_f16_gavgpool_minmax_multipass_ukernel_fn;

xnn_f32_gavgpool_minmax_multipass_ukernel_fn;

xnn_qs8_gavgpool_minmax_multipass_ukernel_fn;

xnn_qu8_gavgpool_minmax_multipass_ukernel_fn;

// AVGPOOL: AVeraGe POOLing single-pass

xnn_avgpool_unipass_ukernel_fn;

xnn_f16_avgpool_minmax_unipass_ukernel_fn;

xnn_f32_avgpool_minmax_unipass_ukernel_fn;

xnn_qu8_avgpool_minmax_unipass_ukernel_fn;

// AVGPOOL: AVeraGe POOLing multi-pass

xnn_avgpool_multipass_ukernel_fn;

xnn_f16_avgpool_minmax_multipass_ukernel_fn;

xnn_f32_avgpool_minmax_multipass_ukernel_fn;

xnn_qu8_avgpool_minmax_multipass_ukernel_fn;

// PAVGPOOL: Pixelwise AVeraGe POOLing single-pass

xnn_pavgpool_unipass_ukernel_fn;

xnn_f16_pavgpool_minmax_unipass_ukernel_fn;

xnn_f32_pavgpool_minmax_unipass_ukernel_fn;

// PAVGPOOL: Pixelwise AVeraGe POOLing multi-pass

xnn_pavgpool_multipass_ukernel_fn;

xnn_f16_pavgpool_minmax_multipass_ukernel_fn;

xnn_f32_pavgpool_minmax_multipass_ukernel_fn;

// MAXPOOL: MAX POOLing

xnn_maxpool_ukernel_fn;

xnn_f16_maxpool_ukernel_fn;

xnn_f32_maxpool_ukernel_fn;

xnn_s8_maxpool_ukernel_fn;

xnn_u8_maxpool_ukernel_fn;

// ARGMAXPOOL: ARG MAX POOLing single-pass

xnn_argmaxpool_unipass_ukernel_fn;

xnn_f32_argmaxpool_unipass_ukernel_fn;

// ARGMAXPOOL: ARG MAX POOLing multi-pass

xnn_argmaxpool_multipass_ukernel_fn;

xnn_f32_argmaxpool_multipass_ukernel_fn;

// UNPOOL: UNPOOLing

xnn_unpool_ukernel_fn;

xnn_x32_unpool_ukernel_fn;

// TRANSPOSEC: TRANSPOSE Constant-size elements

xnn_x8_transposec_ukernel_fn;

xnn_x16_transposec_ukernel_fn;

xnn_x24_transposec_ukernel_fn;

xnn_x32_transposec_ukernel_fn;

xnn_x64_transposec_ukernel_fn;

xnn_transposec_ukernel_fn;

// TRANSPOSEV: TRANSPOSE Variable-size elements

xnn_transposev_ukernel_fn;

// PACKB: PACK B (bias) for GEMM matrix multiplication

xnn_packb_gemm_ukernel_fn;

xnn_x32_packb_gemm_ukernel_fn;

// ZEROB: ZERO B (bias) for GEMM matrix multiplication

xnn_zerob_gemm_ukernel_fn;

xnn_x32_zerob_gemm_ukernel_fn;

// PACKQ: PACK and Quantize (weights) the left-hand operator for GEMM matrix
// multiplication.

xnn_x8_packq_f32qp8_ukernel_fn;

// PACKW: PACK W (weights) for GEMM matrix multiplication
// Weights in GOI layout: Groups, Output channels, Input channels.

xnn_packw_gemm_goi_ukernel_fn;

// TODO - Consolidate packing w/ per_channel and blockwise quant
xnn_packw_gemm_goi_bl_ukernel_fn;

xnn_x8_packw_gemm_goi_ukernel_fn;

xnn_x16_packw_gemm_goi_ukernel_fn;

xnn_x32_packw_gemm_goi_ukernel_fn;

// PACKW: PACK W (weights) for GEMM matrix multiplication
// Weights in GIO layout: Groups, Input channels, Output channels.
xnn_packw_gemm_gio_ukernel_fn;

// PACK: PACK for IGEMM matrix multiplication
// Weights in GOKI layout: Groups, Output channels, Kernel channels, Input channels.
xnn_pack_conv_goki_w_fn;

// PACK: PACK for IGEMM matrix multiplication
// Weights in KGO layout: Kernel channels, groups, Output channels.
xnn_pack_conv_kgo_w_fn;

// PACK: PACK for DECONV SubConv matrix multiplication
// Weights in GOKI layout: Groups, Output channels, Kernel channels, Input channels.
xnn_pack_deconv_goki_w_fn;

// PACKX: PACK X (input) tensor for pre-packed matrix multiplication
xnn_packx_ukernel_fn;

xnn_x32_packx_ukernel_fn;

// FILL: FILL array with value

xnn_fill_ukernel_fn;

// PAD: PAD array with values (fill before, copy array, fill after)

xnn_pad_ukernel_fn;

// REDUCE: Reduce

xnn_reduce_ukernel_fn;

xnn_f16_reduce_ukernel_fn;

xnn_f32_reduce_ukernel_fn;

xnn_u8_reduce_ukernel_fn;

// RDSUM: Discontiguous Reduce-Sum

xnn_rdsum_ukernel_fn;

xnn_f16_f32acc_rdsum_ukernel_fn;

xnn_f32_rdsum_ukernel_fn;

xnn_qs8_rdsum_ukernel_fn;
// RSUM: Reduce-Sum

xnn_f16_rsum_ukernel_fn;

xnn_f16_f32acc_rsum_ukernel_fn;

xnn_f32_rsum_ukernel_fn;

xnn_qs8_rsum_ukernel_fn;

// RMAX: Reduce-MAX

xnn_rmax_ukernel_fn;

xnn_f16_rmax_ukernel_fn;

xnn_f32_rmax_ukernel_fn;

xnn_u8_rmax_ukernel_fn;

// RADDSTOREEXPMINUSMAX: Reduce-ADD & STORE EXP(x_i MINUS MAX[x_i])

xnn_raddstoreexpminusmax_ukernel_fn;

xnn_f16_raddstoreexpminusmax_ukernel_fn;

xnn_f32_raddstoreexpminusmax_ukernel_fn;

// VUNARY: Vector UNARY elementwise

xnn_vunary_ukernel_fn;

// VABS: Vector ABSolute value elementwise

xnn_bf16_vabs_ukernel_fn;

xnn_f16_vabs_ukernel_fn;

xnn_f32_vabs_ukernel_fn;

// VCLAMP: Vector CLAMP elementwise

xnn_f16_vclamp_ukernel_fn;

xnn_f32_vclamp_ukernel_fn;

xnn_s8_vclamp_ukernel_fn;

xnn_u8_vclamp_ukernel_fn;

// VCOPYSIGN: Vector Copysign elementwise

xnn_f32_vcopysign_ukernel_fn;


// MULTIPLY: Vector Multiply (S32 bit)elementwise

xnn_s32_vmul_ukernel_fn;


// VCVT: Vector ConVerT elementwise

xnn_f16_f32_vcvt_ukernel_fn;

xnn_f16_qs8_vcvt_ukernel_fn;

xnn_f32_f16_vcvt_ukernel_fn;

xnn_f32_qs8_vcvt_ukernel_fn;

xnn_f32_qu8_vcvt_ukernel_fn;

xnn_qs8_vcvt_ukernel_fn;

xnn_qs8_f16_vcvt_ukernel_fn;

xnn_qs8_f32_vcvt_ukernel_fn;

xnn_qs16_qs8_vcvt_ukernel_fn;

xnn_qu8_vcvt_ukernel_fn;

xnn_qu8_f32_vcvt_ukernel_fn;

// VELU: Vector Exponential Linear Unit elementwise

xnn_f16_velu_ukernel_fn;

xnn_f32_velu_ukernel_fn;

// VGELU: Vector Gaussian Error Linear Unit elementwise

xnn_f32_vgelu_ukernel_fn;

// VHSWISH: Vector Hard SWISH elementwise

xnn_f16_vhswish_ukernel_fn;

xnn_f32_vhswish_ukernel_fn;

xnn_qs8_vhswish_ukernel_fn;

xnn_qu8_vhswish_ukernel_fn;

// VLRELU: Vector Leaky REctified Linear Unit elementwise

xnn_f16_vlrelu_ukernel_fn;

xnn_f32_vlrelu_ukernel_fn;

xnn_qs8_vlrelu_ukernel_fn;

xnn_qu8_vlrelu_ukernel_fn;

// VNEG: Vector NEGate elementwise

xnn_f16_vneg_ukernel_fn;

xnn_f32_vneg_ukernel_fn;

// VRELU: Vector REctified Linear Unit elementwise

xnn_f32_vrelu_ukernel_fn;

// VRELU generator
xnn_vrelu_generator_fn;

// VROUND: Vector ROUNDing elementwise

xnn_f16_vround_ukernel_fn;

xnn_f32_vround_ukernel_fn;

// VSIGMOID: Vector SIGMOID elementwise

xnn_f16_vsigmoid_ukernel_fn;

xnn_f32_vsigmoid_ukernel_fn;

// VSQR: Vector SQuaRe elementwise

xnn_f16_vsqr_ukernel_fn;

xnn_f32_vsqr_ukernel_fn;

// VSQRT: Vector SQuare RooT elementwise

xnn_f16_vsqrt_ukernel_fn;

xnn_f32_vsqrt_ukernel_fn;

// VSQRTSHIFT: Vector SQuare RooT and SHIFT elementwise

xnn_u64_u32_vsqrtshift_ukernel_fn;

// VRSQRT: Vector Reciprocal SQuare RooT elementwise

xnn_f16_vrsqrt_ukernel_fn;

xnn_f32_vrsqrt_ukernel_fn;

// VTANH: Vector TANH elementwise

xnn_f16_vtanh_ukernel_fn;

xnn_f32_vtanh_ukernel_fn;

// VEXP: Vector Exp elementwise

xnn_f32_vexp_ukernel_fn;

// VLOG: Vector Log elementwise

xnn_f32_vlog_ukernel_fn;

// LUT: vector LookUp Table elementwise

xnn_x8_lut_ukernel_fn;

// LUT32NORM: vector LookUp Table of 32-bit elements and NORMalize elementwise

xnn_u8_lut32norm_ukernel_fn;

// VBINARY: Vector BINARY elementwise

xnn_vbinary_ukernel_fn;

xnn_f16_vbinary_ukernel_fn;

xnn_f32_vbinary_ukernel_fn;

xnn_s32_vbinary_ukernel_fn;

// VBINARY: Vector BINARY elementwise with ReLU activation

xnn_f32_vbinary_relu_ukernel_fn;

// VBINARY: Vector BINARY elementwise with Min+Max activation

xnn_f16_vbinary_minmax_ukernel_fn;

xnn_f32_vbinary_minmax_ukernel_fn;

// VADD: Vector ADD elementwise with Min+Max activation

xnn_qs8_vadd_minmax_ukernel_fn;

xnn_qu8_vadd_minmax_ukernel_fn;

// VMUL: Vector MUL elementwise with Min+Max activation

xnn_qs8_vmul_minmax_ukernel_fn;

xnn_qu8_vmul_minmax_ukernel_fn;


/***************** Microkernel pointers for sparse inference *****************/

// SpMM: Sparse Matrix-Matrix multiplication

xnn_spmm_ukernel_fn;

xnn_f16_spmm_minmax_ukernel_fn;

xnn_f32_spmm_minmax_ukernel_fn;

// CONV-HWC2CHW: direct CONVolution from HWC-layout tensor to CHW-layout tensor

xnn_conv_hwc2chw_ukernel_fn;

xnn_f16_conv_hwc2chw_ukernel_fn;

xnn_f32_conv_hwc2chw_ukernel_fn;

// DWCONV2D-CHW: direct 2D DepthWise CONVolution in CHW layout

xnn_dwconv2d_chw_ukernel_fn;

xnn_f16_dwconv2d_chw_ukernel_fn;

xnn_f32_dwconv2d_chw_ukernel_fn;

// IBILINEAR-CHW: Indirect BILINEAR interpolation in CHW layout

xnn_ibilinear_chw_ukernel_fn;

xnn_f16_ibilinear_chw_ukernel_fn;

xnn_f32_ibilinear_chw_ukernel_fn;

// GAVGPOOL-CW: Global AVeraGe POOLing in CW layout.

xnn_gavgpool_cw_ukernel_fn;

xnn_f16_gavgpool_cw_ukernel_fn;

xnn_f32_gavgpool_cw_ukernel_fn;


/********************* JIT microkernel generator pointers ********************/

xnn_jit_gemm_code_generator_fn;
xnn_jit_igemm_code_generator_fn;


/***************** Audio pre-processing microkernel pointers *****************/

xnn_s16_rmaxabs_ukernel_fn;

xnn_s16_window_ukernel_fn;

xnn_u32_filterbank_accumulate_ukernel_fn;

xnn_u32_filterbank_subtract_ukernel_fn;

xnn_i16_vlshift_ukernel_fn;

xnn_cs16_vsquareabs_ukernel_fn;

xnn_u32_vlog_ukernel_fn;

xnn_cs16_bfly4_ukernel_fn;

xnn_cs16_fftr_ukernel_fn;


/********************* Experimental microkernel pointers *********************/

// ZIPC: ZIP Constant number of arrays

xnn_zipc_ukernel_fn;

xnn_x8_zipc_ukernel_fn;

xnn_x32_zipc_ukernel_fn;

// ZIPV: ZIP Variable number of arrays

xnn_zipv_ukernel_fn;

xnn_x8_zipv_ukernel_fn;

xnn_x32_zipv_ukernel_fn;

// RADDEXPMINUSMAX: Reduce-ADD EXP(x_i MINUS MAX[x_i])

xnn_f32_raddexpminusmax_ukernel_fn;

// VSCALEEXPMINUSMAX: Vector SCALE EXP(x_i MINUS MAX[x_i])

xnn_f32_vscaleexpminusmax_ukernel_fn;

// RADDEXTEXP: Reduce-ADD EXTended ("mantissa" + "exponent") EXPonentials
xnn_f32_raddextexp_ukernel_fn;

// VSCALEEXTEXP: Vector SCALE EXTended ("mantissa" + "exponent") EXPonentials
xnn_f32_vscaleextexp_ukernel_fn;


/***************** Microkernel parameter initializer pointers ****************/

xnn_init_f16_f32_cvt_params_fn;

xnn_init_f16_qs8_cvt_params_fn;

xnn_init_f32_f16_cvt_params_fn;

xnn_init_f32_qs8_cvt_params_fn;

xnn_init_f32_qu8_cvt_params_fn;

xnn_init_qs8_cvt_params_fn;

xnn_init_qs8_f16_cvt_params_fn;

xnn_init_qs8_f32_cvt_params_fn;

xnn_init_qs16_qs8_cvt_params_fn;

xnn_init_qu8_cvt_params_fn;

xnn_init_qu8_f32_cvt_params_fn;

xnn_init_qs8_qc8w_conv_minmax_params_fn;

xnn_init_qs8_conv_minmax_params_fn;

xnn_init_qu8_conv_minmax_params_fn;

xnn_init_qs8_rsum_params_fn;

xnn_init_qs8_avgpool_minmax_params_fn;

xnn_init_qu8_avgpool_minmax_params_fn;

xnn_update_qs8_avgpool_minmax_params_fn;

xnn_update_qu8_avgpool_minmax_params_fn;

xnn_update_f16_gavgpool_neonfp16arith_params_fn;

xnn_init_qs8_add_minmax_params_fn;

xnn_init_qu8_add_minmax_params_fn;

xnn_init_qs8_mul_minmax_params_fn;

xnn_init_qu8_mul_minmax_params_fn;

xnn_init_bf16_abs_params_fn;

xnn_init_f16_abs_params_fn;

xnn_init_f16_default_params_fn;

xnn_init_f32_default_params_fn;
xnn_init_f16_expminus_params_fn;

xnn_init_f32_expminus_params_fn;

xnn_init_s32_default_params_fn;

xnn_init_f16_elu_params_fn;

xnn_init_f32_exp_params_fn;

xnn_init_f32_elu_params_fn;

xnn_init_f16_hswish_params_fn;

xnn_init_f32_hswish_params_fn;

xnn_init_qs8_hswish_params_fn;

xnn_init_qu8_hswish_params_fn;

xnn_init_f16_lrelu_params_fn;

xnn_init_f32_lrelu_params_fn;

xnn_init_f32_log_params_fn;

xnn_init_f32_relu_params_fn;

xnn_init_qs8_lrelu_params_fn;

xnn_init_qu8_lrelu_params_fn;

xnn_init_bf16_minmax_params_fn;

xnn_init_f16_minmax_params_fn;

xnn_init_f32_minmax_params_fn;

xnn_init_f16_qc4w_minmax_params_fn;

xnn_init_f16_qb4w_minmax_params_fn;

xnn_init_f32_qc4w_minmax_params_fn;

xnn_init_f32_qb4w_minmax_params_fn;

xnn_init_s8_minmax_params_fn;

xnn_init_u8_minmax_params_fn;

xnn_init_f16_neg_params_fn;

xnn_init_f16_rnd_params_fn;

xnn_init_f32_rnd_params_fn;

xnn_init_f16_scale_params_fn;

xnn_init_f16_f32acc_scale_params_fn;

xnn_init_f32_scale_params_fn;

xnn_init_f16_scaleminmax_params_fn;

xnn_update_f16_scaleminmax_params_fn;

xnn_init_f32_scaleminmax_params_fn;

xnn_update_f32_scaleminmax_params_fn;

xnn_init_f16_sigmoid_params_fn;

xnn_init_f32_sigmoid_params_fn;

xnn_init_f16_sqrt_params_fn;

xnn_init_f32_sqrt_params_fn;

xnn_init_f16_rsqrt_params_fn;

xnn_init_f32_rsqrt_params_fn;

xnn_init_f16_tanh_params_fn;

xnn_init_f32_tanh_params_fn;

xnn_init_scale_params_fn;

xnn_init_qs8_qc8w_scale_params_fn;

struct xnn_gemm_config;

// Pack weights and biases for GEMM microkernels.
//
// Implementations call the correct packing function selected using flags and
// pack any extra data required using init_extra_data_fns. Accumulators are
// initialized with accumulator_init.
xnn_pack_weights_and_biases_fn;

// Computes the stride of the packing used by a corresponding
// `xnn_pack_weights_and_biases_fn`. The `k_stride` parameter is provided for
// our older packing functions, new wrappers should rely on `gemm_config` and
// `k` instead.
xnn_packed_stride_weights_and_biases_fn;

xnn_init_f16_gavgpool_neon_params_fn;

xnn_init_f32_gavgpool_params_fn;

xnn_update_f32_gavgpool_params_fn;

xnn_init_f32_chw_params_fn;

xnn_init_f16_chw_params_fn;

xnn_update_chw_params_fn;

xnn_update_f32_chw_params_fn;

xnn_update_f16_chw_params_fn;

xnn_init_x8_transpose_params_fn;

xnn_init_x16_transpose_params_fn;

xnn_init_x24_transpose_params_fn;

xnn_init_x32_transpose_params_fn;

xnn_init_x64_transpose_params_fn;

xnn_indirection_init_resize_bilinear2d_hwc_fn;

struct xnn_generated_code_chunk { … };

struct xnn_hmp_dqgemm_ukernel { … };

struct xnn_hmp_dqgemm_bl_ukernel { … };

struct xnn_hmp_gemm_ukernel { … };

struct xnn_hmp_dqigemm_ukernel { … };

struct xnn_hmp_igemm_ukernel { … };

struct xnn_hmp_qp8gemm_ukernel { … };

// Largest GEMM/IGEMM MR used in init.c is 16 (x86 AVX512AMX).
// Largest GEMM/IGEMM MR is 8 in e2e benchmarks.
#define XNN_MAX_MR …

struct gemm_fused_ukernels { … };

#if XNN_PLATFORM_JIT
struct xnn_hmp_gemm_codegen {
  xnn_jit_gemm_code_generator_fn function[XNN_MAX_UARCH_TYPES];
};

struct xnn_hmp_igemm_codegen {
  xnn_jit_igemm_code_generator_fn function[XNN_MAX_UARCH_TYPES];
};

struct gemm_codegens {
  struct xnn_hmp_gemm_codegen gemm[XNN_MAX_MR];
  struct xnn_hmp_igemm_codegen igemm[XNN_MAX_MR];
};
#endif  // XNN_PLATFORM_JIT