chromium/third_party/xnnpack/src/src/operators/transpose-nd.c

// Copyright 2022 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <assert.h>
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

#include "xnnpack.h"
#include "xnnpack/allocator.h"
#include "xnnpack/common.h"
#include "xnnpack/compute.h"
#include "xnnpack/config-types.h"
#include "xnnpack/config.h"
#include "xnnpack/log.h"
#include "xnnpack/microkernel-type.h"
#include "xnnpack/normalization.h"
#include "xnnpack/operator-type.h"
#include "xnnpack/operator.h"
#include "xnnpack/params.h"
#include "pthreadpool.h"

/// Reorder the data in array using the indices in loop_order.
///
/// Changing the loop order can have dramatic performance implications.
static void reorder_array(
    size_t num_dims,
    const size_t loop_order[restrict XNN_MIN_ELEMENTS(1) ],
    size_t array[restrict XNN_MIN_ELEMENTS(1)])
{ … }

static void init_transpose_nd(
    uint32_t flags,
    const struct xnn_transpose_config* transpose_config,
    enum xnn_operator_type operator_type,
    xnn_operator_t transpose_op)
{ … }

static enum xnn_status create_transpose_nd(
    uint32_t flags,
    enum xnn_operator_type operator_type,
    xnn_operator_t* transpose_op_out)
{ … }

enum xnn_status xnn_create_transpose_nd_x64(
  uint32_t flags,
  xnn_operator_t* transpose_op_out)
{ … }

enum xnn_status xnn_create_transpose_nd_x32(
  uint32_t flags,
  xnn_operator_t* transpose_op_out)
{ … }

enum xnn_status xnn_create_transpose_nd_x16(
  uint32_t flags,
  xnn_operator_t* transpose_op_out)
{ … }

enum xnn_status xnn_create_transpose_nd_x8(
  uint32_t flags,
  xnn_operator_t* transpose_op_out)
{ … }

/// input_stride and output_stride are the number of elements between each
/// dimension, not the size of the dimension. This is because depth to space
/// splits the input channel dimension into three dimensions - block_size *
/// block_size * output_channels but gives input_channel_stride the stride over
/// all three dimensions. This must be multiplied by the product of the previous
/// dimensions to get the stride in elements. input_channel_stride is not
/// requried to be a multiple of block_size * block_size * output_channels so
/// the stride in number of elements must be supplied.
/// An interface for sub-tensors can easily be built on top of this.
static enum xnn_status reshape_transpose_nd(
  xnn_operator_t transpose_op,
  const size_t num_dims,
  const size_t* input_shape,
  const size_t* perm,
  const size_t* input_stride,
  const size_t* output_stride,
  size_t element_size)
{ … }

enum xnn_status xnn_reshape_transpose_nd_x64(
    xnn_operator_t transpose_op,
    size_t num_dims,
    const size_t* shape,
    const size_t* perm,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_transpose_nd_x32(
    xnn_operator_t transpose_op,
    size_t num_dims,
    const size_t* shape,
    const size_t* perm,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_transpose_nd_x16(
    xnn_operator_t transpose_op,
    size_t num_dims,
    const size_t* shape,
    const size_t* perm,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_transpose_nd_x8(
    xnn_operator_t transpose_op,
    size_t num_dims,
    const size_t* shape,
    const size_t* perm,
    pthreadpool_t threadpool)
{ … }

/// input_stride and output_stride are the number of elements between each
/// dimension, not the size of the dimension. This is because depth to space
/// splits the input channel dimension into three dimensions - block_size *
/// block_size * output_channels but gives input_channel_stride the stride over
/// all three dimensions. This must be multiplied by the product of the previous
/// dimensions to get the stride in elements. input_channel_stride is not
/// requried to be a multiple of block_size * block_size * output_channels so
/// the stride in number of elements must be supplied.
/// An interface for sub-tensors can easily be built on top of this.
static enum xnn_status setup_transpose_nd(
  xnn_operator_t transpose_op,
  const void* input,
  void* output)
{ … }

enum xnn_status xnn_setup_transpose_nd_x64(
    xnn_operator_t transpose_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_transpose_nd_x32(
    xnn_operator_t transpose_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_transpose_nd_x16(
    xnn_operator_t transpose_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_transpose_nd_x8(
    xnn_operator_t transpose_op,
    const void* input,
    void* output)
{ … }

enum xnn_status run_transpose_nd(
    uint32_t flags,
    const void* input,
    void* output,
    const size_t num_dims,
    const size_t* input_shape,
    const size_t* output_perm,
    size_t element_size,
    enum xnn_operator_type operator_type,
    pthreadpool_t threadpool) { … }

enum xnn_status xnn_run_transpose_nd_x8(
    const void* input,
    void* output,
    const size_t num_dims,
    const size_t* input_shape,
    const size_t* output_perm,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_transpose_nd_x16(
    const void* input,
    void* output,
    const size_t num_dims,
    const size_t* input_shape,
    const size_t* output_perm,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_transpose_nd_x32(
    const void* input,
    void* output,
    const size_t num_dims,
    const size_t* input_shape,
    const size_t* output_perm,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_run_transpose_nd_x64(
    const void* input,
    void* output,
    const size_t num_dims,
    const size_t* input_shape,
    const size_t* output_perm,
    uint32_t flags,
    pthreadpool_t threadpool)
{ … }

enum xnn_status create_depth_to_space_nchw2nhwc(
    uint32_t block_size,
    uint32_t flags,
    enum xnn_operator_type operator_type,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x16(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x32(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status reshape_depth_to_space_nchw2nhwc(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    enum xnn_operator_type operator_type,
    size_t element_size,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out)
{ … }

enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x16(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x32(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status setup_depth_to_space_nchw2nhwc(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x16(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x32(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

static enum xnn_status create_depth_to_space_nhwc(
    uint32_t block_size,
    uint32_t flags,
    enum xnn_operator_type operator_type,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status xnn_create_depth_to_space_nhwc_x8(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status xnn_create_depth_to_space_nhwc_x16(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* depth_to_space_op_out)
{ … }

enum xnn_status xnn_create_depth_to_space_nhwc_x32(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* depth_to_space_op_out)
{ … }

static enum xnn_status reshape_depth_to_space_nhwc(
    xnn_operator_t depth_to_space_op,
    enum xnn_operator_type expected_operator_type,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    uint32_t element_size,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out)
{ … }

enum xnn_status xnn_reshape_depth_to_space_nhwc_x8(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_depth_to_space_nhwc_x16(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_depth_to_space_nhwc_x32(
    xnn_operator_t depth_to_space_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status setup_depth_to_space_nhwc(
    xnn_operator_t depth_to_space_op,
    enum xnn_operator_type expected_operator_type,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_depth_to_space_nhwc_x8(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_depth_to_space_nhwc_x16(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_depth_to_space_nhwc_x32(
    xnn_operator_t depth_to_space_op,
    const void* input,
    void* output)
{ … }

static enum xnn_status create_space_to_depth_nhwc(
    uint32_t block_size,
    uint32_t flags,
    enum xnn_operator_type operator_type,
    xnn_operator_t* space_to_depth_op_out)
{ … }

enum xnn_status xnn_create_space_to_depth_nhwc_x8(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* space_to_depth_op_out)
{ … }

enum xnn_status xnn_create_space_to_depth_nhwc_x16(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* space_to_depth_op_out)
{ … }

enum xnn_status xnn_create_space_to_depth_nhwc_x32(
    uint32_t block_size,
    uint32_t flags,
    xnn_operator_t* space_to_depth_op_out)
{ … }

static enum xnn_status reshape_space_to_depth_nhwc(
    xnn_operator_t space_to_depth_op,
    enum xnn_operator_type expected_operator_type,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    uint32_t element_size,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out)
{ … }

enum xnn_status xnn_reshape_space_to_depth_nhwc_x8(
    xnn_operator_t space_to_depth_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_space_to_depth_nhwc_x16(
    xnn_operator_t space_to_depth_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

enum xnn_status xnn_reshape_space_to_depth_nhwc_x32(
    xnn_operator_t space_to_depth_op,
    size_t batch_size,
    size_t input_height,
    size_t input_width,
    size_t input_channels,
    size_t* output_height_out,
    size_t* output_width_out,
    size_t* output_channels_out,
    pthreadpool_t threadpool)
{ … }

static enum xnn_status setup_space_to_depth_nhwc(
    xnn_operator_t space_to_depth_op,
    enum xnn_operator_type expected_operator_type,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_space_to_depth_nhwc_x8(
    xnn_operator_t space_to_depth_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_space_to_depth_nhwc_x16(
    xnn_operator_t space_to_depth_op,
    const void* input,
    void* output)
{ … }

enum xnn_status xnn_setup_space_to_depth_nhwc_x32(
    xnn_operator_t space_to_depth_op,
    const void* input,
    void* output)
{ … }