| /* -*-arm64-*- |
| * vim: syntax=arm64asm |
| * |
| * AArch64 NEON optimised SAO functions for HEVC decoding |
| * |
| * Copyright (c) 2022 J. Dekker <jdek@itanimul.li> |
| * |
| * This file is part of FFmpeg. |
| * |
| * FFmpeg is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU Lesser General Public |
| * License as published by the Free Software Foundation; either |
| * version 2.1 of the License, or (at your option) any later version. |
| * |
| * FFmpeg is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * Lesser General Public License for more details. |
| * |
| * You should have received a copy of the GNU Lesser General Public |
| * License along with FFmpeg; if not, write to the Free Software |
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| */ |
| |
| #include "libavutil/aarch64/asm.S" |
| |
| #define MAX_PB_SIZE 64 |
| #define AV_INPUT_BUFFER_PADDING_SIZE 64 |
| #define SAO_STRIDE (2*MAX_PB_SIZE + AV_INPUT_BUFFER_PADDING_SIZE) |
| |
| // void sao_band_filter(uint8_t *_dst, uint8_t *_src, |
| // ptrdiff_t stride_dst, ptrdiff_t stride_src, |
| // int16_t *sao_offset_val, int sao_left_class, |
| // int width, int height) |
| function ff_hevc_sao_band_filter_8x8_8_neon, export=1 |
| stp xzr, xzr, [sp, #-64]! |
| stp xzr, xzr, [sp, #16] |
| stp xzr, xzr, [sp, #32] |
| stp xzr, xzr, [sp, #48] |
| mov w8, #4 |
| 0: ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] |
| subs w8, w8, #1 |
| add w10, w8, w5 // k + sao_left_class |
| and w10, w10, #0x1F |
| strh w9, [sp, x10, lsl #1] |
| bne 0b |
| add w6, w6, #7 |
| bic w6, w6, #7 |
| ld1 {v16.16b-v19.16b}, [sp], #64 |
| sub x2, x2, x6 |
| sub x3, x3, x6 |
| movi v20.8h, #1 |
| 1: mov w8, w6 // beginning of line |
| 2: // Simple layout for accessing 16bit values |
| // with 8bit LUT. |
| // |
| // 00 01 02 03 04 05 06 07 |
| // +-----------------------------------> |
| // |xDE#xAD|xCA#xFE|xBE#xEF|xFE#xED|.... |
| // +-----------------------------------> |
| // i-0 i-1 i-2 i-3 |
| ld1 {v2.8b}, [x1], #8 // dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]); |
| subs w8, w8, #8 |
| uxtl v0.8h, v2.8b // load src[x] |
| ushr v2.8h, v0.8h, #3 // >> BIT_DEPTH - 3 |
| shl v1.8h, v2.8h, #1 // low (x2, accessing short) |
| add v3.8h, v1.8h, v20.8h // +1 access upper short |
| sli v1.8h, v3.8h, #8 // shift insert index to upper byte |
| tbx v2.16b, {v16.16b-v19.16b}, v1.16b // table |
| add v1.8h, v0.8h, v2.8h // src[x] + table |
| sqxtun v4.8b, v1.8h // clip + narrow |
| st1 {v4.8b}, [x0], #8 // store |
| // done 8 pixels |
| bne 2b |
| subs w7, w7, #1 // finished line, prep. new |
| add x0, x0, x2 // dst += stride_dst |
| add x1, x1, x3 // src += stride_src |
| bne 1b |
| ret |
| endfunc |
| |
| .Lsao_edge_pos: |
| .word 1 // horizontal |
| .word SAO_STRIDE // vertical |
| .word SAO_STRIDE + 1 // 45 degree |
| .word SAO_STRIDE - 1 // 135 degree |
| |
| // ff_hevc_sao_edge_filter_16x16_8_neon(char *dst, char *src, ptrdiff stride_dst, |
| // int16 *sao_offset_val, int eo, int width, int height) |
| function ff_hevc_sao_edge_filter_16x16_8_neon, export=1 |
| adr x7, .Lsao_edge_pos |
| ld1 {v3.8h}, [x3] // load sao_offset_val |
| add w5, w5, #0xF |
| bic w5, w5, #0xF |
| ldr w4, [x7, w4, uxtw #2] // stride_src |
| mov v3.h[7], v3.h[0] // reorder to [1,2,0,3,4] |
| mov v3.h[0], v3.h[1] |
| mov v3.h[1], v3.h[2] |
| mov v3.h[2], v3.h[7] |
| // split 16bit values into two tables |
| uzp2 v1.16b, v3.16b, v3.16b // sao_offset_val -> upper |
| uzp1 v0.16b, v3.16b, v3.16b // sao_offset_val -> lower |
| movi v2.16b, #2 |
| mov x15, #SAO_STRIDE |
| // strides between end of line and next src/dst |
| sub x15, x15, x5 // stride_src - width |
| sub x16, x2, x5 // stride_dst - width |
| mov x11, x1 // copy base src |
| 1: // new line |
| mov x14, x5 // copy width |
| sub x12, x11, x4 // src_a (prev) = src - sao_edge_pos |
| add x13, x11, x4 // src_b (next) = src + sao_edge_pos |
| 2: // process 16 bytes |
| ld1 {v3.16b}, [x11], #16 // load src |
| ld1 {v4.16b}, [x12], #16 // load src_a (prev) |
| ld1 {v5.16b}, [x13], #16 // load src_b (next) |
| subs x14, x14, #16 |
| cmhi v16.16b, v4.16b, v3.16b // (prev > cur) |
| cmhi v17.16b, v3.16b, v4.16b // (cur > prev) |
| cmhi v18.16b, v5.16b, v3.16b // (next > cur) |
| cmhi v19.16b, v3.16b, v5.16b // (cur > next) |
| sub v20.16b, v16.16b, v17.16b // diff0 = CMP(cur, prev) = (cur > prev) - (cur < prev) |
| sub v21.16b, v18.16b, v19.16b // diff1 = CMP(cur, next) = (cur > next) - (cur < next) |
| add v20.16b, v20.16b, v21.16b // diff = diff0 + diff1 |
| add v20.16b, v20.16b, v2.16b // offset_val = diff + 2 |
| tbl v16.16b, {v0.16b}, v20.16b |
| tbl v17.16b, {v1.16b}, v20.16b |
| uxtl v20.8h, v3.8b // src[0:7] |
| uxtl2 v21.8h, v3.16b // src[7:15] |
| zip1 v18.16b, v16.16b, v17.16b // sao_offset_val lower -> |
| zip2 v19.16b, v16.16b, v17.16b // sao_offset_val upper -> |
| sqadd v20.8h, v18.8h, v20.8h // + sao_offset_val |
| sqadd v21.8h, v19.8h, v21.8h |
| sqxtun v3.8b, v20.8h |
| sqxtun2 v3.16b, v21.8h |
| st1 {v3.16b}, [x0], #16 |
| // filtered 16 bytes |
| b.ne 2b // do we have width to filter? |
| // no width to filter, setup next line |
| subs w6, w6, #1 // filtered line |
| add x11, x11, x15 // stride src to next line |
| add x0, x0, x16 // stride dst to next line |
| b.ne 1b // do we have lines to process? |
| // no lines to filter |
| ret |
| endfunc |
| |
| // ff_hevc_sao_edge_filter_8x8_8_neon(char *dst, char *src, ptrdiff stride_dst, |
| // int16 *sao_offset_val, int eo, int width, int height) |
| function ff_hevc_sao_edge_filter_8x8_8_neon, export=1 |
| adr x7, .Lsao_edge_pos |
| ldr w4, [x7, w4, uxtw #2] |
| ld1 {v3.8h}, [x3] |
| mov v3.h[7], v3.h[0] |
| mov v3.h[0], v3.h[1] |
| mov v3.h[1], v3.h[2] |
| mov v3.h[2], v3.h[7] |
| uzp2 v1.16b, v3.16b, v3.16b |
| uzp1 v0.16b, v3.16b, v3.16b |
| movi v2.16b, #2 |
| add x16, x0, x2 |
| lsl x2, x2, #1 |
| mov x15, #SAO_STRIDE |
| mov x8, x1 |
| sub x9, x1, x4 |
| add x10, x1, x4 |
| 1: ld1 {v3.d}[0], [ x8], x15 |
| ld1 {v4.d}[0], [ x9], x15 |
| ld1 {v5.d}[0], [x10], x15 |
| ld1 {v3.d}[1], [ x8], x15 |
| ld1 {v4.d}[1], [ x9], x15 |
| ld1 {v5.d}[1], [x10], x15 |
| subs w6, w6, #2 |
| cmhi v16.16b, v4.16b, v3.16b |
| cmhi v17.16b, v3.16b, v4.16b |
| cmhi v18.16b, v5.16b, v3.16b |
| cmhi v19.16b, v3.16b, v5.16b |
| sub v20.16b, v16.16b, v17.16b |
| sub v21.16b, v18.16b, v19.16b |
| add v20.16b, v20.16b, v21.16b |
| add v20.16b, v20.16b, v2.16b |
| tbl v16.16b, {v0.16b}, v20.16b |
| tbl v17.16b, {v1.16b}, v20.16b |
| uxtl v20.8h, v3.8b |
| uxtl2 v21.8h, v3.16b |
| zip1 v18.16b, v16.16b, v17.16b |
| zip2 v19.16b, v16.16b, v17.16b |
| sqadd v20.8h, v18.8h, v20.8h |
| sqadd v21.8h, v19.8h, v21.8h |
| sqxtun v6.8b, v20.8h |
| sqxtun v7.8b, v21.8h |
| st1 {v6.8b}, [ x0], x2 |
| st1 {v7.8b}, [x16], x2 |
| b.ne 1b |
| ret |
| endfunc |