chromium/third_party/ruy/src/ruy/pack_x86.h

/* Copyright 2019 Google LLC. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifndef RUY_RUY_PACK_X86_H_
#define RUY_RUY_PACK_X86_H_

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <type_traits>

#include "ruy/check_macros.h"
#include "ruy/mat.h"
#include "ruy/opt_set.h"
#include "ruy/pack_common.h"
#include "ruy/path.h"
#include "ruy/platform.h"
#include "ruy/profiler/instrumentation.h"
#include "ruy/tune.h"

ruy  // namespace ruy

#if ((RUY_PLATFORM_AVX || RUY_PLATFORM_AVX2_FMA) && RUY_OPT(ASM))

#include <immintrin.h>  // IWYU pragma: keep

namespace ruy {
namespace {

template <Path path>
inline __m256 Mm256UnpackloPsx2(const __m256 a, const __m256 b) { … }

template <Path path>
inline __m256 Mm256UnpackhiPsx2(const __m256 a, const __m256 b) { … }

template <Path path>
inline __m256i CompareGreaterThan(const __m256i&, const __m256i&) { … }

// Shared between AVX and AVX2+FMA.
template <Path path>
inline __m256i MaskLoadu(int available_src_rows, std::int8_t zero_point,
                         const std::int8_t* addr) { … }

}  // namespace.

template <typename PackImpl, Path path>
inline void PackFloatColMajorForAvxCommonPacker(const float* src_ptr,
                                                const float* zerobuf,
                                                int src_stride,
                                                int remaining_src_cols,
                                                int src_rows, float* packed_ptr,
                                                float* trailing_buf) { … }
}  // namespace ruy
#endif  //  (RUY_PLATFORM_AVX || RUY_PLATFORM_AVX2_FMA) && RUY_OPT(ASM)

#endif  // RUY_RUY_PACK_X86_H_