gemm-config.c | Explore in Territory

// Copyright 2023 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <assert.h>
#include <stddef.h>

#if _WIN32
  #include <windows.h>
#else
  #include <pthread.h>
#endif

#if XNN_ENABLE_CPUINFO
  #include <cpuinfo.h>
#endif  // XNN_ENABLE_CPUINFO

#include "xnnpack/common.h"
#include "xnnpack/config.h"
#include "xnnpack/gemm.h"
#include "xnnpack/hardware-config.h"
#include "xnnpack/igemm.h"
#include "xnnpack/microfnptr.h"
#include "xnnpack/microparams-init.h"
#include "xnnpack/pack.h"
#include "xnnpack/packw.h"

#if XNN_ARCH_WASMSIMD
  #include <emscripten.h>
#endif

#define XNN_MR_TO_INDEX(MR) …


static struct xnn_gemm_config f16_gemm_config = …;
static struct xnn_gemm_config f32_gemm_config = …;
static struct xnn_gemm_config f32_gemm_nr2_config = …;
static struct xnn_gemm_config f32_qc4w_gemm_config = …;
static struct xnn_gemm_config f32_qc8w_gemm_config = …;
static struct xnn_gemm_config qd8_f16_qb4w_gemm_config = …;
static struct xnn_gemm_config qd8_f16_qc4w_gemm_config = …;
static struct xnn_gemm_config qd8_f16_qc8w_gemm_config = …;
static struct xnn_gemm_config qd8_f32_qb4w_gemm_config = …;
static struct xnn_gemm_config qd8_f32_qc4w_gemm_config = …;
static struct xnn_gemm_config qd8_f32_qc8w_gemm_config = …;
static struct xnn_gemm_config qp8_f32_qc4w_gemm_config = …;
static struct xnn_gemm_config qs8_qc8w_gemm_config = …;
static struct xnn_gemm_config qu8_gemm_config = …;

#if XNN_PLATFORM_WINDOWS
  static INIT_ONCE init_guard_f16_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_f32_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_f32_gemm_nr2 = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_f32_qc4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_f32_qc8w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f16_qb4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f16_qc4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f16_qc8w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f32_qb4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f32_qc4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qd8_f32_qc8w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qp8_f32_qc4w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qs8_qc8w_gemm = INIT_ONCE_STATIC_INIT;
  static INIT_ONCE init_guard_qu8_gemm = INIT_ONCE_STATIC_INIT;
#else
  static pthread_once_t init_guard_f16_gemm = …;
  static pthread_once_t init_guard_f32_gemm = …;
  static pthread_once_t init_guard_f32_gemm_nr2 = …;
  static pthread_once_t init_guard_f32_qc4w_gemm = …;
  static pthread_once_t init_guard_f32_qc8w_gemm = …;
  static pthread_once_t init_guard_qd8_f16_qb4w_gemm = …;
  static pthread_once_t init_guard_qd8_f16_qc4w_gemm = …;
  static pthread_once_t init_guard_qd8_f16_qc8w_gemm = …;
  static pthread_once_t init_guard_qd8_f32_qb4w_gemm = …;
  static pthread_once_t init_guard_qd8_f32_qc4w_gemm = …;
  static pthread_once_t init_guard_qd8_f32_qc8w_gemm = …;
  static pthread_once_t init_guard_qp8_f32_qc4w_gemm = …;
  static pthread_once_t init_guard_qs8_qc8w_gemm = …;
  static pthread_once_t init_guard_qu8_gemm = …;
#endif

static void init_f16_gemm_config(void) { … }

#if XNN_ARCH_WASMSIMD
  EM_JS(int, hardware_concurrency, (void), {
    var concurrency = 1;
    try {
      concurrency = self.navigator.hardwareConcurrency;
    } catch(e) {
      // d8 environment doesn't provide `self`, thus we keep the default
    }
    return concurrency;
  });
  // A cpu with more than `kCoreCountThresholdForAdaptiveAvxOptimization` is
  // assumed to support AVX instructions.
  const int kCoreCountThresholdForAdaptiveAvxOptimization = 4;
#endif

static void init_f32_gemm_config(void) { … }

static void init_f32_gemm_nr2_config(void) { … }

static void init_f32_qc4w_gemm_config(void) { … }

static void init_f32_qc8w_gemm_config(void) { … }

static void init_qd8_f16_qc4w_gemm_config(void) { … }

static void init_qd8_f16_qb4w_gemm_config(void) { … }

static void init_qd8_f32_qc4w_gemm_config(void) { … }

static void init_qp8_f32_qc4w_gemm_config(void) { … }

static void init_qd8_f32_qb4w_gemm_config(void) { … }

static void init_qd8_f16_qc8w_gemm_config(void) { … }

static void init_qd8_f32_qc8w_gemm_config(void) { … }

static void init_qs8_qc8w_gemm_config(void) { … }

static void init_qu8_gemm_config(void) { … }

#if XNN_PLATFORM_WINDOWS
  static BOOL CALLBACK init_f16_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_f16_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_f32_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_f32_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_f32_gemm_nr2_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_f32_gemm_nr2_config();
    return TRUE;
  }

  static BOOL CALLBACK init_f32_qc4w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_f32_qc4w_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_f32_qc8w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_f32_qc8w_gemm_config();
    return TRUE;
  }

 static BOOL CALLBACK init_qd8_f16_qc8w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f16_qc8w_gemm_config();
    return TRUE;
 }

 static BOOL CALLBACK init_qd8_f16_qc4w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f16_qc4w_gemm_config();
    return TRUE;
  }

 static BOOL CALLBACK init_qd8_f16_qb4w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f16_qb4w_gemm_config();
    return TRUE;
  }

 static BOOL CALLBACK init_qd8_f32_qc4w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f32_qc4w_gemm_config();
    return TRUE;
  }

 static BOOL CALLBACK init_qd8_f32_qb4w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f32_qb4w_gemm_config();
    return TRUE;
  }

 static BOOL CALLBACK init_qd8_f32_qc8w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qd8_f32_qc8w_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_qp8_f32_qc4w_gemm_config_windows(
      PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qp8_f32_qc4w_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_qs8_qc8w_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qs8_qc8w_gemm_config();
    return TRUE;
  }

  static BOOL CALLBACK init_qu8_gemm_config_windows(PINIT_ONCE init_once, PVOID parameter, PVOID* context) {
    init_qu8_gemm_config();
    return TRUE;
  }
#endif

struct xnn_gemm_config* xnn_init_f16_gemm_config() { … }

struct xnn_gemm_config* xnn_init_f32_gemm_config() { … }

struct xnn_gemm_config* xnn_init_f32_gemm_nr2_config() { … }

struct xnn_gemm_config* xnn_init_f32_qc4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_f32_qc8w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f16_qc8w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f16_qc4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f16_qb4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f32_qc4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f32_qb4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qd8_f32_qc8w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qp8_f32_qc4w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qs8_qc8w_gemm_config() { … }

struct xnn_gemm_config* xnn_init_qu8_gemm_config() { … }
chromium/third_party/xnnpack/src/src/configs/gemm-config.c
