chromium/third_party/xnnpack/src/src/operators/unary-elementwise-nc.c

// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <assert.h>
#include <inttypes.h>
#include <math.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include <fp16/fp16.h>
#include "xnnpack.h"
#include "xnnpack/allocator.h"
#include "xnnpack/common.h"
#include "xnnpack/compute.h"
#include "xnnpack/config-types.h"
#include "xnnpack/config.h"
#include "xnnpack/log.h"
#include "xnnpack/microfnptr.h"
#include "xnnpack/microparams.h"
#include "xnnpack/operator-type.h"
#include "xnnpack/operator.h"
#include "xnnpack/params.h"
#include "pthreadpool.h"

static xnn_status_t check_op_type(xnn_operator_t op,
                                  enum xnn_operator_type expected_type) { … }

static void init_unary_elementwise_nc(
    uint32_t flags,
    const void* params,
    size_t params_size,
    enum xnn_operator_type operator_type,
    const struct xnn_unary_elementwise_config* unary_elementwise_config,
    const struct xnn_reduce_config* rminmax_config,
    xnn_operator_t unary_elementwise_op)
{ … }

static enum xnn_status create_unary_elementwise_nc(
    uint32_t flags,
    const struct xnn_unary_elementwise_config* unary_elementwise_config,
    const struct xnn_reduce_config* rminmax_config,
    const void* params,
    size_t params_size,
    enum xnn_operator_type operator_type,
    xnn_operator_t* unary_elementwise_op_out)
{ … }

static bool is_copy_operator(enum xnn_operator_type operator_type) { … }

static enum xnn_status reshape_unary_elementwise_nc(
    xnn_operator_t unary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    uint32_t log2_input_size,
    uint32_t log2_output_size,
    const void* params,
    size_t params_size,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status setup_unary_elementwise_nc(
    xnn_operator_t unary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_create_abs_nc_f16(
    uint32_t flags,
    xnn_operator_t* abs_op_out)
{ … }

enum xnn_status xnn_create_abs_nc_f32(
    uint32_t flags,
    xnn_operator_t* abs_op_out)
{ … }

enum xnn_status xnn_create_bankers_rounding_nc_f16(
    uint32_t flags,
    xnn_operator_t* rounding_op_out)
{ … }

enum xnn_status xnn_create_bankers_rounding_nc_f32(
    uint32_t flags,
    xnn_operator_t* rounding_op_out)
{ … }

enum xnn_status xnn_create_ceiling_nc_f16(
    uint32_t flags,
    xnn_operator_t* ceiling_op_out)
{ … }

enum xnn_status xnn_create_ceiling_nc_f32(
    uint32_t flags,
    xnn_operator_t* ceiling_op_out)
{ … }

enum xnn_status xnn_create_clamp_nc_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* clamp_op_out)
{ … }

enum xnn_status xnn_create_clamp_nc_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* clamp_op_out)
{ … }

enum xnn_status xnn_create_clamp_nc_s8(
    int8_t output_min,
    int8_t output_max,
    uint32_t flags,
    xnn_operator_t* clamp_op_out)
{ … }

enum xnn_status xnn_create_clamp_nc_u8(
    uint8_t output_min,
    uint8_t output_max,
    uint32_t flags,
    xnn_operator_t* clamp_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f16_f32(
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f32_f16(
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f32_qs8(
  float output_scale,
  int8_t output_zero_point,
  int8_t output_min,
  int8_t output_max,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f16_qd8(
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f32_qd8(
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_f32_qp8(uint32_t flags,
                                              xnn_operator_t* convert_op_out) { … }

enum xnn_status xnn_create_convert_nc_f32_qu8(
  float output_scale,
  uint8_t output_zero_point,
  uint8_t output_min,
  uint8_t output_max,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qs8(
  float input_scale,
  int8_t input_zero_point,
  float output_scale,
  int8_t output_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qs8_f16(
  float input_scale,
  int8_t input_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qs8_f32(
  float input_scale,
  int8_t input_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qs16_qs8(
  float input_scale,
  float output_scale,
  int8_t output_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qu8(
  float input_scale,
  uint8_t input_zero_point,
  float output_scale,
  uint8_t output_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_convert_nc_qu8_f32(
  float input_scale,
  uint8_t input_zero_point,
  uint32_t flags,
  xnn_operator_t* convert_op_out)
{ … }

enum xnn_status xnn_create_copy_nc_x8(
    uint32_t flags,
    xnn_operator_t* copy_op_out)
{ … }

enum xnn_status xnn_create_copy_nc_x16(
    uint32_t flags,
    xnn_operator_t* copy_op_out)
{ … }

enum xnn_status xnn_create_copy_nc_x32(
    uint32_t flags,
    xnn_operator_t* copy_op_out)
{ … }

enum xnn_status xnn_create_elu_nc_f16(
  float alpha,
  uint32_t flags,
  xnn_operator_t* elu_op_out)
{ … }

enum xnn_status xnn_create_elu_nc_f32(
  float alpha,
  uint32_t flags,
  xnn_operator_t* elu_op_out)
{ … }

enum xnn_status xnn_create_floor_nc_f16(
    uint32_t flags,
    xnn_operator_t* floor_op_out)
{ … }

enum xnn_status xnn_create_floor_nc_f32(
    uint32_t flags,
    xnn_operator_t* floor_op_out)
{ … }

enum xnn_status xnn_create_gelu_nc_f32(uint32_t flags,
                                       xnn_operator_t* gelu_op_out) { … }

enum xnn_status xnn_create_hardswish_nc_f16(
    uint32_t flags,
    xnn_operator_t* hardswish_op_out)
{ … }

enum xnn_status xnn_create_hardswish_nc_f32(
    uint32_t flags,
    xnn_operator_t* hardswish_op_out)
{ … }

enum xnn_status xnn_create_leaky_relu_nc_f16(
  float negative_slope,
  uint32_t flags,
  xnn_operator_t* leaky_relu_op_out)
{ … }

enum xnn_status xnn_create_leaky_relu_nc_f32(
  float negative_slope,
  uint32_t flags,
  xnn_operator_t* leaky_relu_op_out)
{ … }

enum xnn_status xnn_create_log_nc_f32(
  uint32_t flags,
  xnn_operator_t* log_op_out)
{ … }

enum xnn_status xnn_reshape_log_nc_f32(
  xnn_operator_t log_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_setup_log_nc_f32(
  xnn_operator_t log_op,
  const float* input,
  float* output)
{ … }

enum xnn_status xnn_create_leaky_relu_nc_qs8(
  float negative_slope,
  int8_t input_zero_point,
  float input_scale,
  int8_t output_zero_point,
  float output_scale,
  uint32_t flags,
  xnn_operator_t* leaky_relu_op_out)
{ … }

enum xnn_status xnn_create_leaky_relu_nc_qu8(
  float negative_slope,
  uint8_t input_zero_point,
  float input_scale,
  uint8_t output_zero_point,
  float output_scale,
  uint32_t flags,
  xnn_operator_t* leaky_relu_op_out)
{ … }

enum xnn_status xnn_create_negate_nc_f16(
    uint32_t flags,
    xnn_operator_t* negate_op_out)
{ … }

enum xnn_status xnn_create_negate_nc_f32(
    uint32_t flags,
    xnn_operator_t* negate_op_out)
{ … }

enum xnn_status xnn_create_sigmoid_nc_f16(
    uint32_t flags,
    xnn_operator_t* sigmoid_op_out)
{ … }

enum xnn_status xnn_create_sigmoid_nc_f32(
    uint32_t flags,
    xnn_operator_t* sigmoid_op_out)
{ … }

enum xnn_status xnn_create_square_nc_f16(
    uint32_t flags,
    xnn_operator_t* square_op_out)
{ … }

enum xnn_status xnn_create_square_nc_f32(
    uint32_t flags,
    xnn_operator_t* square_op_out)
{ … }

enum xnn_status xnn_create_square_root_nc_f16(
    uint32_t flags,
    xnn_operator_t* sqrt_op_out)
{ … }

enum xnn_status xnn_create_square_root_nc_f32(
    uint32_t flags,
    xnn_operator_t* sqrt_op_out)
{ … }

enum xnn_status xnn_create_reciprocal_square_root_nc_f16(
    uint32_t flags,
    xnn_operator_t* rsqrt_op_out)
{ … }

enum xnn_status xnn_create_reciprocal_square_root_nc_f32(
    uint32_t flags, xnn_operator_t* rsqrt_op_out) { … }

enum xnn_status xnn_create_tanh_nc_f16(
    uint32_t flags,
    xnn_operator_t* tanh_op_out)
{ … }

enum xnn_status xnn_create_tanh_nc_f32(
    uint32_t flags,
    xnn_operator_t* tanh_op_out)
{ … }

enum xnn_status xnn_create_truncation_nc_f16(
    uint32_t flags,
    xnn_operator_t* truncation_op_out)
{ … }

enum xnn_status xnn_create_truncation_nc_f32(
    uint32_t flags,
    xnn_operator_t* truncation_op_out)
{ … }

enum xnn_status xnn_reshape_abs_nc_f16(
    xnn_operator_t abs_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_abs_nc_f32(
    xnn_operator_t abs_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_bankers_rounding_nc_f16(
    xnn_operator_t rounding_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_bankers_rounding_nc_f32(
    xnn_operator_t rounding_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_ceiling_nc_f16(
    xnn_operator_t ceiling_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_ceiling_nc_f32(
    xnn_operator_t ceiling_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_clamp_nc_f16(
    xnn_operator_t clamp_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_clamp_nc_f32(
    xnn_operator_t clamp_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_clamp_nc_s8(
    xnn_operator_t clamp_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_clamp_nc_u8(
    xnn_operator_t clamp_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f16_f32(
  xnn_operator_t convert_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f32_f16(
  xnn_operator_t convert_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f16_qd8(
    xnn_operator_t convert_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f32_qd8(
    xnn_operator_t convert_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f32_qp8(xnn_operator_t convert_op,
                                               size_t batch_size,
                                               size_t channels,
                                               size_t input_stride,
                                               pthreadpool_t threadpool) { … }

enum xnn_status xnn_reshape_convert_nc_f32_qs8(
  xnn_operator_t convert_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_f32_qu8(
  xnn_operator_t convert_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qs8(
  xnn_operator_t convert_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qs16_qs8(
  xnn_operator_t convert_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qs8_f16(
  xnn_operator_t convert_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qs8_f32(
  xnn_operator_t convert_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qu8(
  xnn_operator_t convert_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_convert_nc_qu8_f32(
  xnn_operator_t convert_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_copy_nc_x8(
    xnn_operator_t copy_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_copy_nc_x16(
    xnn_operator_t copy_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_copy_nc_x32(
    xnn_operator_t copy_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_elu_nc_f16(
    xnn_operator_t elu_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_elu_nc_f32(
    xnn_operator_t elu_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_create_exp_nc_f32(
  uint32_t flags,
  xnn_operator_t* exp_op_out)
{ … }

enum xnn_status xnn_reshape_exp_nc_f32(
  xnn_operator_t exp_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_setup_exp_nc_f32(
  xnn_operator_t exp_op,
  const float* input,
  float* output)
{ … }

enum xnn_status xnn_reshape_floor_nc_f16(
    xnn_operator_t floor_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_floor_nc_f32(
    xnn_operator_t floor_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_gelu_nc_f32(xnn_operator_t gelu_op,
                                        size_t batch_size, size_t channels,
                                        size_t input_stride,
                                        size_t output_stride,
                                        pthreadpool_t threadpool) { … }

enum xnn_status xnn_reshape_hardswish_nc_f16(
    xnn_operator_t hardswish_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_hardswish_nc_f32(
    xnn_operator_t hardswish_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_leaky_relu_nc_f16(
  xnn_operator_t leaky_relu_op,
  size_t batch_size,
    size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_leaky_relu_nc_f32(
  xnn_operator_t leaky_relu_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_leaky_relu_nc_qs8(
  xnn_operator_t leaky_relu_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_leaky_relu_nc_qu8(
  xnn_operator_t leaky_relu_op,
  size_t batch_size,
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_negate_nc_f16(
    xnn_operator_t negate_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_negate_nc_f32(
    xnn_operator_t negate_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_reciprocal_square_root_nc_f16(
    xnn_operator_t rsqrt_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_reciprocal_square_root_nc_f32(
    xnn_operator_t rsqrt_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_sigmoid_nc_f16(
    xnn_operator_t sigmoid_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_sigmoid_nc_f32(
    xnn_operator_t sigmoid_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_square_nc_f16(
    xnn_operator_t square_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_square_nc_f32(
    xnn_operator_t square_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_square_root_nc_f16(
    xnn_operator_t sqrt_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_square_root_nc_f32(
    xnn_operator_t sqrt_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_tanh_nc_f16(
    xnn_operator_t tanh_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_tanh_nc_f32(
    xnn_operator_t tanh_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_truncation_nc_f16(
    xnn_operator_t truncation_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_truncation_nc_f32(
    xnn_operator_t truncation_op,
    size_t batch_size,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_setup_abs_nc_f16(
    xnn_operator_t abs_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_abs_nc_f32(
    xnn_operator_t abs_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_bankers_rounding_nc_f16(
    xnn_operator_t rounding_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_bankers_rounding_nc_f32(
    xnn_operator_t rounding_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_ceiling_nc_f16(
    xnn_operator_t ceiling_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_ceiling_nc_f32(
    xnn_operator_t ceiling_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_clamp_nc_f16(
    xnn_operator_t clamp_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_clamp_nc_f32(
    xnn_operator_t clamp_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_clamp_nc_s8(
    xnn_operator_t clamp_op,
    const int8_t* input,
    int8_t* output)
{ … }

enum xnn_status xnn_setup_clamp_nc_u8(
    xnn_operator_t clamp_op,
    const uint8_t* input,
    uint8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_f16_f32(
  xnn_operator_t convert_op,
  const void* input,
  float* output)
{ … }

enum xnn_status xnn_setup_convert_nc_f32_f16(
  xnn_operator_t convert_op,
  const float* input,
  void* output)
{ … }

enum xnn_status xnn_setup_convert_nc_f16_qd8(
  xnn_operator_t convert_op,
  const void* input,
  int8_t* output,
  struct xnn_dynamic_quantization_params* quantization_params)
{ … }

enum xnn_status xnn_setup_convert_nc_f32_qd8(
  xnn_operator_t convert_op,
  const float* input,
  int8_t* output,
  struct xnn_dynamic_quantization_params* quantization_params)
{ … }

enum xnn_status xnn_setup_convert_nc_f32_qp8(xnn_operator_t convert_op,
                                             const float* input,
                                             int8_t* output) { … }

enum xnn_status xnn_setup_convert_nc_f32_qs8(
  xnn_operator_t convert_op,
  const float* input,
  int8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_f32_qu8(
  xnn_operator_t convert_op,
  const float* input,
  uint8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qs8(
  xnn_operator_t convert_op,
  const int8_t* input,
  int8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qs16_qs8(
  xnn_operator_t convert_op,
  const int16_t* input,
  int8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qs8_f16(
  xnn_operator_t convert_op,
  const int8_t* input,
  void* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qs8_f32(
  xnn_operator_t convert_op,
  const int8_t* input,
  float* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qu8(
  xnn_operator_t convert_op,
  const uint8_t* input,
  uint8_t* output)
{ … }

enum xnn_status xnn_setup_convert_nc_qu8_f32(
  xnn_operator_t convert_op,
  const uint8_t* input,
  float* output)
{ … }

enum xnn_status xnn_setup_copy_nc_x8(
    xnn_operator_t copy_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_copy_nc_x16(
    xnn_operator_t copy_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_copy_nc_x32(
    xnn_operator_t copy_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_elu_nc_f16(
    xnn_operator_t elu_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_elu_nc_f32(
    xnn_operator_t elu_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_floor_nc_f16(
    xnn_operator_t floor_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_floor_nc_f32(
    xnn_operator_t floor_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_gelu_nc_f32(xnn_operator_t gelu_op,
                                      const float* input, float* output) { … }

enum xnn_status xnn_setup_hardswish_nc_f16(
    xnn_operator_t hardswish_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_hardswish_nc_f32(
    xnn_operator_t hardswish_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_leaky_relu_nc_f16(
  xnn_operator_t leaky_relu_op,
  const void* input,
  void* output)
{ … }

enum xnn_status xnn_setup_leaky_relu_nc_f32(
  xnn_operator_t leaky_relu_op,
  const float* input,
  float* output)
{ … }

enum xnn_status xnn_setup_leaky_relu_nc_qs8(
  xnn_operator_t leaky_relu_op,
  const int8_t* input,
  int8_t* output)
{ … }

enum xnn_status xnn_setup_leaky_relu_nc_qu8(
  xnn_operator_t leaky_relu_op,
  const uint8_t* input,
  uint8_t* output)
{ … }

enum xnn_status xnn_setup_negate_nc_f16(
    xnn_operator_t negate_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_negate_nc_f32(
    xnn_operator_t negate_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_reciprocal_square_root_nc_f16(
    xnn_operator_t rsqrt_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_reciprocal_square_root_nc_f32(
    xnn_operator_t rsqrt_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_sigmoid_nc_f16(
    xnn_operator_t sigmoid_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_sigmoid_nc_f32(
    xnn_operator_t sigmoid_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_square_nc_f16(
    xnn_operator_t square_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_square_nc_f32(
    xnn_operator_t square_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_square_root_nc_f16(
    xnn_operator_t sqrt_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_square_root_nc_f32(
    xnn_operator_t sqrt_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_tanh_nc_f16(
    xnn_operator_t tanh_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_tanh_nc_f32(
    xnn_operator_t tanh_op,
    const float* input,
    float* output)
{ … }

enum xnn_status xnn_setup_truncation_nc_f16(
    xnn_operator_t truncation_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_truncation_nc_f32(
    xnn_operator_t truncation_op,
    const float* input,
    float* output)
{ … }

static enum xnn_status run_unary_elementwise_nc(
    enum xnn_operator_type operator_type,
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const void* input,
    void* output,
    const struct xnn_unary_elementwise_config* unary_elementwise_config,
    const void* params,
    size_t params_size,
    uint32_t log2_input_size,
    uint32_t log2_output_size,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_abs_nc_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    float* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_bankers_rounding_nc_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    float* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_ceiling_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_clamp_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  float output_min,
  float output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_f16_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const void* input,
    float* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_f32_f16(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    void* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_f32_qs8(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    int8_t* output,
    float output_scale,
    int8_t output_zero_point,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_f32_qu8(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    uint8_t* output,
    float output_scale,
    uint8_t output_zero_point,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_qs8_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const int8_t* input,
    float* output,
    float input_scale,
    int8_t input_zero_point,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_qs16_qs8(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const int16_t* input,
  int8_t* output,
  float input_scale,
  float output_scale,
  int8_t output_zero_point,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_convert_nc_qu8_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const uint8_t* input,
    float* output,
    float input_scale,
    uint8_t input_zero_point,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_copy_nc_x32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const uint32_t* input,
    uint32_t* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_elu_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  float alpha,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_floor_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_gelu_nc_f32(size_t channels, size_t input_stride,
                                    size_t output_stride, size_t batch_size,
                                    const float* input, float* output,
                                    uint32_t flags, pthreadpool_t threadpool) { … }

enum xnn_status xnn_run_hardswish_nc_f32(
    size_t channels,
    size_t input_stride,
    size_t output_stride,
    size_t batch_size,
    const float* input,
    float* output,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_leaky_relu_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  float negative_slope,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_negate_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_reciprocal_square_root_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_sigmoid_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_square_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_square_root_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_tanh_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_truncation_nc_f32(
  size_t channels,
  size_t input_stride,
  size_t output_stride,
  size_t batch_size,
  const float* input,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }