// Copyright 2015 Google Inc. All Rights Reserved. // // Use of this source code is governed by a BSD-style license // that can be found in the COPYING file in the root of the source // tree. An additional intellectual property rights grant can be found // in the file PATENTS. All contributing project authors may // be found in the AUTHORS file in the root of the source tree. // ----------------------------------------------------------------------------- // // NEON version of rescaling functions // // Author: Skal ([email protected]) #include "src/dsp/dsp.h" #if defined(WEBP_USE_NEON) && !defined(WEBP_REDUCE_SIZE) #include <arm_neon.h> #include <assert.h> #include "src/dsp/neon.h" #include "src/utils/rescaler_utils.h" #define ROUNDER … #define MULT_FIX_C … #define MULT_FIX_FLOOR_C … #define LOAD_32x4 … #define LOAD_32x8 … #define STORE_32x8 … #if (WEBP_RESCALER_RFIX == 32) #define MAKE_HALF_CST … // note: B is actualy scale>>1. See MAKE_HALF_CST #define MULT_FIX … #define MULT_FIX_FLOOR … #else #error "MULT_FIX/WEBP_RESCALER_RFIX need some more work" #endif static uint32x4_t Interpolate_NEON(const rescaler_t* const frow, const rescaler_t* const irow, uint32_t A, uint32_t B) { LOAD_32x4(frow, A0); LOAD_32x4(irow, B0); const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A); const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A); const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B); const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B); const uint32x4_t E = vcombine_u32( vrshrn_n_u64(D0, WEBP_RESCALER_RFIX), vrshrn_n_u64(D1, WEBP_RESCALER_RFIX)); return E; } static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) { int x_out; uint8_t* const dst = wrk->dst; rescaler_t* const irow = wrk->irow; const int x_out_max = wrk->dst_width * wrk->num_channels; const int max_span = x_out_max & ~7; const rescaler_t* const frow = wrk->frow; const uint32_t fy_scale = wrk->fy_scale; const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale); assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(wrk->y_expand); assert(wrk->y_sub != 0); if (wrk->y_accum == 0) { for (x_out = 0; x_out < max_span; x_out += 8) { LOAD_32x4(frow + x_out + 0, A0); LOAD_32x4(frow + x_out + 4, A1); const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half); const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half); const uint16x4_t C0 = vmovn_u32(B0); const uint16x4_t C1 = vmovn_u32(B1); const uint8x8_t D = vqmovn_u16(vcombine_u16(C0, C1)); vst1_u8(dst + x_out, D); } for (; x_out < x_out_max; ++x_out) { const uint32_t J = frow[x_out]; const int v = (int)MULT_FIX_C(J, fy_scale); dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } else { const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); for (x_out = 0; x_out < max_span; x_out += 8) { const uint32x4_t C0 = Interpolate_NEON(frow + x_out + 0, irow + x_out + 0, A, B); const uint32x4_t C1 = Interpolate_NEON(frow + x_out + 4, irow + x_out + 4, A, B); const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half); const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half); const uint16x4_t E0 = vmovn_u32(D0); const uint16x4_t E1 = vmovn_u32(D1); const uint8x8_t F = vqmovn_u16(vcombine_u16(E0, E1)); vst1_u8(dst + x_out, F); } for (; x_out < x_out_max; ++x_out) { const uint64_t I = (uint64_t)A * frow[x_out] + (uint64_t)B * irow[x_out]; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX_C(J, fy_scale); dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) { int x_out; uint8_t* const dst = wrk->dst; rescaler_t* const irow = wrk->irow; const int x_out_max = wrk->dst_width * wrk->num_channels; const int max_span = x_out_max & ~7; const rescaler_t* const frow = wrk->frow; const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); const uint32_t fxy_scale = wrk->fxy_scale; const uint32x4_t zero = vdupq_n_u32(0); const int32x4_t yscale_half = MAKE_HALF_CST(yscale); const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale); assert(!WebPRescalerOutputDone(wrk)); assert(wrk->y_accum <= 0); assert(!wrk->y_expand); if (yscale) { for (x_out = 0; x_out < max_span; x_out += 8) { LOAD_32x8(frow + x_out, in0, in1); LOAD_32x8(irow + x_out, in2, in3); const uint32x4_t A0 = MULT_FIX_FLOOR(in0, yscale_half); const uint32x4_t A1 = MULT_FIX_FLOOR(in1, yscale_half); const uint32x4_t B0 = vqsubq_u32(in2, A0); const uint32x4_t B1 = vqsubq_u32(in3, A1); const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half); const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half); const uint16x4_t D0 = vmovn_u32(C0); const uint16x4_t D1 = vmovn_u32(C1); const uint8x8_t E = vqmovn_u16(vcombine_u16(D0, D1)); vst1_u8(dst + x_out, E); STORE_32x8(A0, A1, irow + x_out); } for (; x_out < x_out_max; ++x_out) { const uint32_t frac = (uint32_t)MULT_FIX_FLOOR_C(frow[x_out], yscale); const int v = (int)MULT_FIX_C(irow[x_out] - frac, fxy_scale); dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = frac; // new fractional start } } else { for (x_out = 0; x_out < max_span; x_out += 8) { LOAD_32x8(irow + x_out, in0, in1); const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half); const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half); const uint16x4_t B0 = vmovn_u32(A0); const uint16x4_t B1 = vmovn_u32(A1); const uint8x8_t C = vqmovn_u16(vcombine_u16(B0, B1)); vst1_u8(dst + x_out, C); STORE_32x8(zero, zero, irow + x_out); } for (; x_out < x_out_max; ++x_out) { const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale); dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = 0; } } } #undef MULT_FIX_FLOOR_C #undef MULT_FIX_C #undef MULT_FIX_FLOOR #undef MULT_FIX #undef ROUNDER //------------------------------------------------------------------------------ extern void WebPRescalerDspInitNEON(void); WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) { WebPRescalerExportRowExpand = RescalerExportRowExpand_NEON; WebPRescalerExportRowShrink = RescalerExportRowShrink_NEON; } #else // !WEBP_USE_NEON WEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON) #endif // WEBP_USE_NEON
Codebase Browser
godot