/* * Copyright (c) 2016, Alliance for Open Media. All rights reserved. * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License * was not distributed with this source code in the LICENSE file, you can * obtain it at www.aomedia.org/license/software. If the Alliance for Open * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ #include <assert.h> #include <emmintrin.h> // SSE2 #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" #include "aom_dsp/blend.h" #include "aom_dsp/x86/mem_sse2.h" #include "aom_dsp/x86/synonyms.h" #include "aom_ports/mem.h" #if !CONFIG_REALTIME_ONLY unsigned int aom_get_mb_ss_sse2(const int16_t *src) { __m128i vsum = _mm_setzero_si128(); int i; for (i = 0; i < 32; ++i) { const __m128i v = xx_loadu_128(src); vsum = _mm_add_epi32(vsum, _mm_madd_epi16(v, v)); src += 8; } vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 8)); vsum = _mm_add_epi32(vsum, _mm_srli_si128(vsum, 4)); return (unsigned int)_mm_cvtsi128_si32(vsum); } #endif // !CONFIG_REALTIME_ONLY static inline __m128i load4x2_sse2(const uint8_t *const p, const int stride) { … } static inline __m128i load8_8to16_sse2(const uint8_t *const p) { … } static inline void load16_8to16_sse2(const uint8_t *const p, __m128i *out) { … } // Accumulate 4 32bit numbers in val to 1 32bit number static inline unsigned int add32x4_sse2(__m128i val) { … } // Accumulate 8 16bit in sum to 4 32bit number static inline __m128i sum_to_32bit_sse2(const __m128i sum) { … } static inline void variance_kernel_sse2(const __m128i src, const __m128i ref, __m128i *const sse, __m128i *const sum) { … } // Can handle 128 pixels' diff sum (such as 8x16 or 16x8) // Slightly faster than variance_final_256_pel_sse2() // diff sum of 128 pixels can still fit in 16bit integer static inline void variance_final_128_pel_sse2(__m128i vsse, __m128i vsum, unsigned int *const sse, int *const sum) { … } // Can handle 256 pixels' diff sum (such as 16x16) static inline void variance_final_256_pel_sse2(__m128i vsse, __m128i vsum, unsigned int *const sse, int *const sum) { … } // Can handle 512 pixels' diff sum (such as 16x32 or 32x16) static inline void variance_final_512_pel_sse2(__m128i vsse, __m128i vsum, unsigned int *const sse, int *const sum) { … } // Can handle 1024 pixels' diff sum (such as 32x32) static inline void variance_final_1024_pel_sse2(__m128i vsse, __m128i vsum, unsigned int *const sse, int *const sum) { … } static inline void variance4_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } static inline void variance8_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } static inline void variance16_kernel_sse2(const uint8_t *const src, const uint8_t *const ref, __m128i *const sse, __m128i *const sum) { … } static inline void variance16_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } static inline void variance32_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } static inline void variance64_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } static inline void variance128_sse2(const uint8_t *src, const int src_stride, const uint8_t *ref, const int ref_stride, const int h, __m128i *const sse, __m128i *const sum) { … } void aom_get_var_sse_sum_8x8_quad_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse8x8, int *sum8x8, unsigned int *tot_sse, int *tot_sum, uint32_t *var8x8) { … } void aom_get_var_sse_sum_16x16_dual_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse16x16, unsigned int *tot_sse, int *tot_sum, uint32_t *var16x16) { … } #define AOM_VAR_NO_LOOP_SSE2(bw, bh, bits, max_pixels) … AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) AOM_VAR_NO_LOOP_SSE2(…) #if !CONFIG_REALTIME_ONLY AOM_VAR_NO_LOOP_SSE2(4, 16, 6, 128) AOM_VAR_NO_LOOP_SSE2(16, 4, 6, 128) AOM_VAR_NO_LOOP_SSE2(8, 32, 8, 256) AOM_VAR_NO_LOOP_SSE2(32, 8, 8, 256) AOM_VAR_NO_LOOP_SSE2(16, 64, 10, 1024) #endif #define AOM_VAR_LOOP_SSE2(bw, bh, bits, uh) … AOM_VAR_LOOP_SSE2(32, 64, 11, 32) // 32x32 * ( 64/32 ) AOM_VAR_LOOP_SSE2(64, 32, 11, 16) // 64x16 * ( 32/16 ) AOM_VAR_LOOP_SSE2(64, 64, 12, 16) // 64x16 * ( 64/16 ) AOM_VAR_LOOP_SSE2(64, 128, 13, 16) // 64x16 * ( 128/16 ) AOM_VAR_LOOP_SSE2(128, 64, 13, 8) // 128x8 * ( 64/8 ) AOM_VAR_LOOP_SSE2(128, 128, 14, 8) // 128x8 * ( 128/8 ) #if !CONFIG_REALTIME_ONLY AOM_VAR_NO_LOOP_SSE2(64, 16, 10, 1024) #endif unsigned int aom_mse8x8_sse2(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, unsigned int *sse) { … } unsigned int aom_mse8x16_sse2(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, unsigned int *sse) { … } unsigned int aom_mse16x8_sse2(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, unsigned int *sse) { … } unsigned int aom_mse16x16_sse2(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, unsigned int *sse) { … } #if CONFIG_AV1_HIGHBITDEPTH static inline __m128i highbd_comp_mask_pred_line_sse2(const __m128i s0, const __m128i s1, const __m128i a) { const __m128i alpha_max = _mm_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS)); const __m128i round_const = _mm_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1); const __m128i a_inv = _mm_sub_epi16(alpha_max, a); const __m128i s_lo = _mm_unpacklo_epi16(s0, s1); const __m128i a_lo = _mm_unpacklo_epi16(a, a_inv); const __m128i pred_lo = _mm_madd_epi16(s_lo, a_lo); const __m128i pred_l = _mm_srai_epi32(_mm_add_epi32(pred_lo, round_const), AOM_BLEND_A64_ROUND_BITS); const __m128i s_hi = _mm_unpackhi_epi16(s0, s1); const __m128i a_hi = _mm_unpackhi_epi16(a, a_inv); const __m128i pred_hi = _mm_madd_epi16(s_hi, a_hi); const __m128i pred_h = _mm_srai_epi32(_mm_add_epi32(pred_hi, round_const), AOM_BLEND_A64_ROUND_BITS); const __m128i comp = _mm_packs_epi32(pred_l, pred_h); return comp; } void aom_highbd_comp_mask_pred_sse2(uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask) { int i = 0; uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8); uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); const uint16_t *src0 = invert_mask ? pred : ref; const uint16_t *src1 = invert_mask ? ref : pred; const int stride0 = invert_mask ? width : ref_stride; const int stride1 = invert_mask ? ref_stride : width; const __m128i zero = _mm_setzero_si128(); if (width == 8) { do { const __m128i s0 = _mm_loadu_si128((const __m128i *)(src0)); const __m128i s1 = _mm_loadu_si128((const __m128i *)(src1)); const __m128i m_8 = _mm_loadl_epi64((const __m128i *)mask); const __m128i m_16 = _mm_unpacklo_epi8(m_8, zero); const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m_16); _mm_storeu_si128((__m128i *)comp_pred, comp); src0 += stride0; src1 += stride1; mask += mask_stride; comp_pred += width; i += 1; } while (i < height); } else if (width == 16) { do { const __m128i s0 = _mm_loadu_si128((const __m128i *)(src0)); const __m128i s2 = _mm_loadu_si128((const __m128i *)(src0 + 8)); const __m128i s1 = _mm_loadu_si128((const __m128i *)(src1)); const __m128i s3 = _mm_loadu_si128((const __m128i *)(src1 + 8)); const __m128i m_8 = _mm_loadu_si128((const __m128i *)mask); const __m128i m01_16 = _mm_unpacklo_epi8(m_8, zero); const __m128i m23_16 = _mm_unpackhi_epi8(m_8, zero); const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m01_16); const __m128i comp1 = highbd_comp_mask_pred_line_sse2(s2, s3, m23_16); _mm_storeu_si128((__m128i *)comp_pred, comp); _mm_storeu_si128((__m128i *)(comp_pred + 8), comp1); src0 += stride0; src1 += stride1; mask += mask_stride; comp_pred += width; i += 1; } while (i < height); } else { do { for (int x = 0; x < width; x += 32) { for (int j = 0; j < 2; j++) { const __m128i s0 = _mm_loadu_si128((const __m128i *)(src0 + x + j * 16)); const __m128i s2 = _mm_loadu_si128((const __m128i *)(src0 + x + 8 + j * 16)); const __m128i s1 = _mm_loadu_si128((const __m128i *)(src1 + x + j * 16)); const __m128i s3 = _mm_loadu_si128((const __m128i *)(src1 + x + 8 + j * 16)); const __m128i m_8 = _mm_loadu_si128((const __m128i *)(mask + x + j * 16)); const __m128i m01_16 = _mm_unpacklo_epi8(m_8, zero); const __m128i m23_16 = _mm_unpackhi_epi8(m_8, zero); const __m128i comp = highbd_comp_mask_pred_line_sse2(s0, s1, m01_16); const __m128i comp1 = highbd_comp_mask_pred_line_sse2(s2, s3, m23_16); _mm_storeu_si128((__m128i *)(comp_pred + j * 16), comp); _mm_storeu_si128((__m128i *)(comp_pred + 8 + j * 16), comp1); } comp_pred += 32; } src0 += stride0; src1 += stride1; mask += mask_stride; i += 1; } while (i < height); } } #endif // CONFIG_AV1_HIGHBITDEPTH static uint64_t mse_4xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src, int sstride, int h) { … } static uint64_t mse_8xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src, int sstride, int h) { … } uint64_t aom_mse_wxh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src, int sstride, int w, int h) { … } uint64_t aom_mse_16xh_16bit_sse2(uint8_t *dst, int dstride, uint16_t *src, int w, int h) { … }
Codebase Browser
chromium