chromium/third_party/xnnpack/src/src/operators/binary-elementwise-nd.c

// Copyright 2019 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <assert.h>
#include <inttypes.h>
#include <math.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include <fp16/fp16.h>
#include "xnnpack.h"
#include "xnnpack/allocator.h"
#include "xnnpack/common.h"
#include "xnnpack/compute.h"
#include "xnnpack/config-types.h"
#include "xnnpack/config.h"
#include "xnnpack/log.h"
#include "xnnpack/math.h"
#include "xnnpack/microparams.h"
#include "xnnpack/operator-type.h"
#include "xnnpack/operator.h"
#include "xnnpack/params.h"
#include "pthreadpool.h"

static void init_binary_elementwise_nd(
  const void* params,
  const void* params2,
  size_t params_size,
  uint32_t flags,
  enum xnn_operator_type operator_type,
  const struct xnn_binary_elementwise_subconfig* binary_elementwise_subconfig,
  xnn_operator_t binary_elementwise_op)
{ … }

static enum xnn_status create_binary_elementwise_nd(
    uint32_t flags,
    const void* params,
    const void* params2,
    size_t params_size,
    enum xnn_operator_type operator_type,
    const struct xnn_binary_elementwise_subconfig* binary_elementwise_subconfig,
    xnn_operator_t* binary_elementwise_op_out)
{ … }

static enum xnn_status create_binary_elementwise_nd_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    enum xnn_operator_type operator_type,
    const struct xnn_binary_elementwise_config* config,
    xnn_operator_t* binary_elementwise_op_out)
{ … }

static enum xnn_status create_binary_elementwise_nd_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    enum xnn_operator_type operator_type,
    const struct xnn_binary_elementwise_config* config,
    xnn_operator_t* binary_elementwise_op_out)
{ … }

enum xnn_status xnn_create_add_nd_qs8(
    int8_t input1_zero_point,
    float input1_scale,
    int8_t input2_zero_point,
    float input2_scale,
    int8_t output_zero_point,
    float output_scale,
    int8_t output_min,
    int8_t output_max,
    uint32_t flags,
    xnn_operator_t* add_op_out)
{ … }

enum xnn_status xnn_create_add_nd_qu8(
    uint8_t input1_zero_point,
    float input1_scale,
    uint8_t input2_zero_point,
    float input2_scale,
    uint8_t output_zero_point,
    float output_scale,
    uint8_t output_min,
    uint8_t output_max,
    uint32_t flags,
    xnn_operator_t* add_op_out)
{ … }

enum xnn_status xnn_create_add_nd_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* add_op_out)
{ … }

enum xnn_status xnn_create_add_nd_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* add_op_out)
{ … }

enum xnn_status xnn_create_divide_nd_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* divide_op_out)
{ … }

enum xnn_status xnn_create_copysign_nd_f32(
    uint32_t flags,
    xnn_operator_t* copysign_op_out)
{ … }

enum xnn_status xnn_create_divide_nd_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* divide_op_out)
{ … }

enum xnn_status xnn_create_maximum_nd_f16(
    uint32_t flags,
    xnn_operator_t* maximum_op_out)
{ … }

enum xnn_status xnn_create_maximum_nd_f32(
    uint32_t flags,
    xnn_operator_t* maximum_op_out)
{ … }

enum xnn_status xnn_create_minimum_nd_f16(
    uint32_t flags,
    xnn_operator_t* minimum_op_out)
{ … }

enum xnn_status xnn_create_minimum_nd_f32(
    uint32_t flags,
    xnn_operator_t* minimum_op_out)
{ … }

enum xnn_status xnn_create_multiply_nd_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* multiply_op_out)
{ … }

enum xnn_status xnn_create_multiply_nd_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* multiply_op_out)
{ … }

enum xnn_status xnn_create_multiply_nd_qs8(
    int8_t input1_zero_point,
    float input1_scale,
    int8_t input2_zero_point,
    float input2_scale,
    int8_t output_zero_point,
    float output_scale,
    int8_t output_min,
    int8_t output_max,
    uint32_t flags,
    xnn_operator_t* multiply_op_out)
{ … }

enum xnn_status xnn_create_multiply_nd_qu8(
    uint8_t input1_zero_point,
    float input1_scale,
    uint8_t input2_zero_point,
    float input2_scale,
    uint8_t output_zero_point,
    float output_scale,
    uint8_t output_min,
    uint8_t output_max,
    uint32_t flags,
    xnn_operator_t* multiply_op_out)
{ … }

enum xnn_status xnn_create_squared_difference_nd_f16(
    uint32_t flags,
    xnn_operator_t* squared_difference_op_out)
{ … }

enum xnn_status xnn_create_squared_difference_nd_f32(
    uint32_t flags,
    xnn_operator_t* squared_difference_op_out)
{ … }


enum xnn_status xnn_create_multiply_nd_s32(
    uint32_t flags,
    xnn_operator_t* multiply_op_out)
{ … }


enum xnn_status xnn_create_subtract_nd_f16(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* subtract_op_out)
{ … }

enum xnn_status xnn_create_subtract_nd_f32(
    float output_min,
    float output_max,
    uint32_t flags,
    xnn_operator_t* subtract_op_out)
{ … }

enum xnn_status xnn_create_subtract_nd_qs8(
    int8_t input1_zero_point,
    float input1_scale,
    int8_t input2_zero_point,
    float input2_scale,
    int8_t output_zero_point,
    float output_scale,
    int8_t output_min,
    int8_t output_max,
    uint32_t flags,
    xnn_operator_t* subtract_op_out)
{ … }

enum xnn_status xnn_create_subtract_nd_qu8(
    uint8_t input1_zero_point,
    float input1_scale,
    uint8_t input2_zero_point,
    float input2_scale,
    uint8_t output_zero_point,
    float output_scale,
    uint8_t output_min,
    uint8_t output_max,
    uint32_t flags,
    xnn_operator_t* subtract_op_out)
{ … }

static enum xnn_status reshape_binary_elementwise_nd(
    xnn_operator_t binary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    uint32_t log2_element_size,
    const void* params,
    size_t params_size,
    const void* reversed_params,
    size_t reversed_params_size,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status reshape_binary_elementwise_nd_f16(
    xnn_operator_t binary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status reshape_binary_elementwise_nd_f32(
    xnn_operator_t binary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_add_nd_f16(
    xnn_operator_t add_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_add_nd_f32(
    xnn_operator_t add_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_add_nd_qs8(
    xnn_operator_t add_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_add_nd_qu8(
    xnn_operator_t add_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_divide_nd_f16(
    xnn_operator_t divide_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_divide_nd_f32(
    xnn_operator_t divide_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_copysign_nd_f32(
    xnn_operator_t copysign_op,
    size_t num_mag_dims,
    const size_t* mag_shape,
    size_t num_sign_dims,
    const size_t* sign_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_maximum_nd_f16(
    xnn_operator_t maximum_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_maximum_nd_f32(
    xnn_operator_t maximum_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_minimum_nd_f16(
    xnn_operator_t minimum_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_minimum_nd_f32(
    xnn_operator_t minimum_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }


enum xnn_status xnn_reshape_multiply_nd_f16(
    xnn_operator_t multiply_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_multiply_nd_f32(
    xnn_operator_t multiply_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_multiply_nd_qs8(
    xnn_operator_t multiply_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_multiply_nd_qu8(
    xnn_operator_t multiply_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_squared_difference_nd_f16(
    xnn_operator_t squared_difference_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_squared_difference_nd_f32(
    xnn_operator_t squared_difference_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }


enum xnn_status xnn_reshape_multiply_nd_s32(
    xnn_operator_t mul_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_subtract_nd_f16(
    xnn_operator_t subtract_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_subtract_nd_f32(
    xnn_operator_t subtract_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_subtract_nd_qs8(
    xnn_operator_t subtract_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_subtract_nd_qu8(
    xnn_operator_t subtract_op,
    size_t num_input1_dims,
    const size_t* input1_shape,
    size_t num_input2_dims,
    const size_t* input2_shape,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status setup_binary_elementwise_nd(
    xnn_operator_t binary_elementwise_op,
    enum xnn_operator_type expected_operator_type,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_add_nd_f16(
    xnn_operator_t add_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_add_nd_f32(
    xnn_operator_t add_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_add_nd_qs8(
    xnn_operator_t add_op,
    const int8_t* input1,
    const int8_t* input2,
    int8_t* output)
{ … }

enum xnn_status xnn_setup_add_nd_qu8(
    xnn_operator_t add_op,
    const uint8_t* input1,
    const uint8_t* input2,
    uint8_t* output)
{ … }

enum xnn_status xnn_setup_divide_nd_f16(
    xnn_operator_t divide_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_divide_nd_f32(
    xnn_operator_t divide_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_copysign_nd_f32(
    xnn_operator_t copysign_op,
    const float* mag,
    const float* sign,
    float* output)
{ … }

enum xnn_status xnn_setup_maximum_nd_f16(
    xnn_operator_t maximum_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_maximum_nd_f32(
    xnn_operator_t maximum_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_minimum_nd_f16(
    xnn_operator_t minimum_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_minimum_nd_f32(
    xnn_operator_t minimum_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_multiply_nd_f16(
    xnn_operator_t multiply_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_multiply_nd_f32(
    xnn_operator_t multiply_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_multiply_nd_qs8(
    xnn_operator_t multiply_op,
    const int8_t* input1,
    const int8_t* input2,
    int8_t* output)
{ … }

enum xnn_status xnn_setup_multiply_nd_qu8(
    xnn_operator_t multiply_op,
    const uint8_t* input1,
    const uint8_t* input2,
    uint8_t* output)
{ … }

enum xnn_status xnn_setup_squared_difference_nd_f16(
    xnn_operator_t squared_difference_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }

enum xnn_status xnn_setup_squared_difference_nd_f32(
    xnn_operator_t squared_difference_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_subtract_nd_f16(
    xnn_operator_t subtract_op,
    const void* input1,
    const void* input2,
    void* output)
{ … }


enum xnn_status xnn_setup_multiply_nd_s32(
    xnn_operator_t mul_op,
    const int32_t* input1,
    const int32_t* input2,
    int32_t* output)
{ … }


enum xnn_status xnn_setup_subtract_nd_f32(
    xnn_operator_t subtract_op,
    const float* input1,
    const float* input2,
    float* output)
{ … }

enum xnn_status xnn_setup_subtract_nd_qs8(
    xnn_operator_t subtract_op,
    const int8_t* input1,
    const int8_t* input2,
    int8_t* output)
{ … }

enum xnn_status xnn_setup_subtract_nd_qu8(
    xnn_operator_t subtract_op,
    const uint8_t* input1,
    const uint8_t* input2,
    uint8_t* output)
{ … }

static enum xnn_status run_binary_elementwise_nd(
  enum xnn_operator_type operator_type,
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const void* input1,
  const void* input2,
  void* output,
  uint32_t log2_element_size,
  size_t params_offset,
  size_t setup_params_size,
  size_t rparams_offset,
  size_t setup_reversed_params_size,
  const struct xnn_binary_elementwise_subconfig* binary_elementwise_subconfig,
  const void* create_params,
  const void* create_params2,
  size_t create_params_size,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

static enum xnn_status run_binary_elementwise_nd_f32(
  enum xnn_operator_type operator_type,
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  float output_min,
  float output_max,
  const struct xnn_binary_elementwise_config* config,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_add_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  float output_min,
  float output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_divide_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  float output_min,
  float output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_maximum_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_minimum_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_multiply_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  float output_min,
  float output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_subtract_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  float output_min,
  float output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_squared_difference_nd_f32(
  size_t num_input1_dims,
  const size_t* input1_shape,
  size_t num_input2_dims,
  const size_t* input2_shape,
  const float* input1,
  const float* input2,
  float* output,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }


enum xnn_status xnn_run_add_nd_qs8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  int8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  int8_t input2_zero_point,
  float input2_scale,
  const int8_t* input1,
  const int8_t* input2,
  int8_t* output,
  int8_t output_zero_point,
  float output_scale,
  int8_t output_min,
  int8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_multiply_nd_qs8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  int8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  int8_t input2_zero_point,
  float input2_scale,
  const int8_t* input1,
  const int8_t* input2,
  int8_t* output,
  int8_t output_zero_point,
  float output_scale,
  int8_t output_min,
  int8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_subtract_nd_qs8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  int8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  int8_t input2_zero_point,
  float input2_scale,
  const int8_t* input1,
  const int8_t* input2,
  int8_t* output,
  int8_t output_zero_point,
  float output_scale,
  int8_t output_min,
  int8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_add_nd_qu8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  uint8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  uint8_t input2_zero_point,
  float input2_scale,
  const uint8_t* input1,
  const uint8_t* input2,
  uint8_t* output,
  uint8_t output_zero_point,
  float output_scale,
  uint8_t output_min,
  uint8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_multiply_nd_qu8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  uint8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  uint8_t input2_zero_point,
  float input2_scale,
  const uint8_t* input1,
  const uint8_t* input2,
  uint8_t* output,
  uint8_t output_zero_point,
  float output_scale,
  uint8_t output_min,
  uint8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_subtract_nd_qu8(
  size_t num_input1_dims,
  const size_t* input1_shape,
  uint8_t input1_zero_point,
  float input1_scale,
  size_t num_input2_dims,
  const size_t* input2_shape,
  uint8_t input2_zero_point,
  float input2_scale,
  const uint8_t* input1,
  const uint8_t* input2,
  uint8_t* output,
  uint8_t output_zero_point,
  float output_scale,
  uint8_t output_min,
  uint8_t output_max,
  uint32_t flags,
  pthreadpool_t threadpool)
{ … }