| /* |
| * AltiVec optimizations for libjpeg-turbo |
| * |
| * Copyright (C) 2015, D. R. Commander. All Rights Reserved. |
| * |
| * This software is provided 'as-is', without any express or implied |
| * warranty. In no event will the authors be held liable for any damages |
| * arising from the use of this software. |
| * |
| * Permission is granted to anyone to use this software for any purpose, |
| * including commercial applications, and to alter it and redistribute it |
| * freely, subject to the following restrictions: |
| * |
| * 1. The origin of this software must not be misrepresented; you must not |
| * claim that you wrote the original software. If you use this software |
| * in a product, an acknowledgment in the product documentation would be |
| * appreciated but is not required. |
| * 2. Altered source versions must be plainly marked as such, and must not be |
| * misrepresented as being the original software. |
| * 3. This notice may not be removed or altered from any source distribution. |
| */ |
| |
| /* CHROMA UPSAMPLING */ |
| |
| #include "jsimd_altivec.h" |
| |
| |
| void jsimd_h2v1_fancy_upsample_altivec(int max_v_samp_factor, |
| JDIMENSION downsampled_width, |
| JSAMPARRAY input_data, |
| JSAMPARRAY *output_data_ptr) |
| { |
| JSAMPARRAY output_data = *output_data_ptr; |
| JSAMPROW inptr, outptr; |
| int inrow, incol; |
| |
| __vector unsigned char this0, last0, p_last0, next0 = { 0 }, p_next0, |
| out; |
| __vector short this0e, this0o, this0l, this0h, last0l, last0h, |
| next0l, next0h, outle, outhe, outlo, outho; |
| |
| /* Constants */ |
| __vector unsigned char pb_zero = { __16X(0) }, pb_three = { __16X(3) }, |
| last_index_col0 = |
| { 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, |
| last_index = |
| { 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 }, |
| next_index = |
| { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }, |
| next_index_lastcol = |
| { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15 }, |
| #if __BIG_ENDIAN__ |
| merge_pack_index = |
| { 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31 }; |
| #else |
| merge_pack_index = |
| { 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30 }; |
| #endif |
| __vector short pw_one = { __8X(1) }, pw_two = { __8X(2) }; |
| |
| for (inrow = 0; inrow < max_v_samp_factor; inrow++) { |
| inptr = input_data[inrow]; |
| outptr = output_data[inrow]; |
| |
| if (downsampled_width & 15) |
| inptr[downsampled_width] = inptr[downsampled_width - 1]; |
| |
| this0 = vec_ld(0, inptr); |
| p_last0 = vec_perm(this0, this0, last_index_col0); |
| last0 = this0; |
| |
| for (incol = downsampled_width; incol > 0; |
| incol -= 16, inptr += 16, outptr += 32) { |
| |
| if (downsampled_width - incol > 0) { |
| p_last0 = vec_perm(last0, this0, last_index); |
| last0 = this0; |
| } |
| |
| if (incol <= 16) |
| p_next0 = vec_perm(this0, this0, next_index_lastcol); |
| else { |
| next0 = vec_ld(16, inptr); |
| p_next0 = vec_perm(this0, next0, next_index); |
| } |
| |
| this0e = (__vector short)vec_mule(this0, pb_three); |
| this0o = (__vector short)vec_mulo(this0, pb_three); |
| this0l = vec_mergeh(this0e, this0o); |
| this0h = vec_mergel(this0e, this0o); |
| |
| last0l = (__vector short)VEC_UNPACKHU(p_last0); |
| last0h = (__vector short)VEC_UNPACKLU(p_last0); |
| last0l = vec_add(last0l, pw_one); |
| |
| next0l = (__vector short)VEC_UNPACKHU(p_next0); |
| next0h = (__vector short)VEC_UNPACKLU(p_next0); |
| next0l = vec_add(next0l, pw_two); |
| |
| outle = vec_add(this0l, last0l); |
| outlo = vec_add(this0l, next0l); |
| outle = vec_sr(outle, (__vector unsigned short)pw_two); |
| outlo = vec_sr(outlo, (__vector unsigned short)pw_two); |
| |
| out = vec_perm((__vector unsigned char)outle, |
| (__vector unsigned char)outlo, merge_pack_index); |
| vec_st(out, 0, outptr); |
| |
| if (incol > 8) { |
| last0h = vec_add(last0h, pw_one); |
| next0h = vec_add(next0h, pw_two); |
| |
| outhe = vec_add(this0h, last0h); |
| outho = vec_add(this0h, next0h); |
| outhe = vec_sr(outhe, (__vector unsigned short)pw_two); |
| outho = vec_sr(outho, (__vector unsigned short)pw_two); |
| |
| out = vec_perm((__vector unsigned char)outhe, |
| (__vector unsigned char)outho, merge_pack_index); |
| vec_st(out, 16, outptr); |
| } |
| |
| this0 = next0; |
| } |
| } |
| } |
| |
| |
| void jsimd_h2v2_fancy_upsample_altivec(int max_v_samp_factor, |
| JDIMENSION downsampled_width, |
| JSAMPARRAY input_data, |
| JSAMPARRAY *output_data_ptr) |
| { |
| JSAMPARRAY output_data = *output_data_ptr; |
| JSAMPROW inptr_1, inptr0, inptr1, outptr0, outptr1; |
| int inrow, outrow, incol; |
| |
| __vector unsigned char this_1, this0, this1, out; |
| __vector short this_1l, this_1h, this0l, this0h, this1l, this1h, |
| lastcolsum_1h, lastcolsum1h, |
| p_lastcolsum_1l, p_lastcolsum_1h, p_lastcolsum1l, p_lastcolsum1h, |
| thiscolsum_1l, thiscolsum_1h, thiscolsum1l, thiscolsum1h, |
| nextcolsum_1l = { 0 }, nextcolsum_1h = { 0 }, |
| nextcolsum1l = { 0 }, nextcolsum1h = { 0 }, |
| p_nextcolsum_1l, p_nextcolsum_1h, p_nextcolsum1l, p_nextcolsum1h, |
| tmpl, tmph, outle, outhe, outlo, outho; |
| |
| /* Constants */ |
| __vector unsigned char pb_zero = { __16X(0) }, |
| last_index_col0 = |
| { 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, |
| last_index = |
| { 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 }, |
| next_index = |
| { 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }, |
| next_index_lastcol = |
| { 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 15 }, |
| #if __BIG_ENDIAN__ |
| merge_pack_index = |
| { 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31 }; |
| #else |
| merge_pack_index = |
| { 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30 }; |
| #endif |
| __vector short pw_zero = { __8X(0) }, pw_three = { __8X(3) }, |
| pw_seven = { __8X(7) }, pw_eight = { __8X(8) }; |
| __vector unsigned short pw_four = { __8X(4) }; |
| |
| for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) { |
| |
| inptr_1 = input_data[inrow - 1]; |
| inptr0 = input_data[inrow]; |
| inptr1 = input_data[inrow + 1]; |
| outptr0 = output_data[outrow++]; |
| outptr1 = output_data[outrow++]; |
| |
| if (downsampled_width & 15) { |
| inptr_1[downsampled_width] = inptr_1[downsampled_width - 1]; |
| inptr0[downsampled_width] = inptr0[downsampled_width - 1]; |
| inptr1[downsampled_width] = inptr1[downsampled_width - 1]; |
| } |
| |
| this0 = vec_ld(0, inptr0); |
| this0l = (__vector short)VEC_UNPACKHU(this0); |
| this0h = (__vector short)VEC_UNPACKLU(this0); |
| this0l = vec_mladd(this0l, pw_three, pw_zero); |
| this0h = vec_mladd(this0h, pw_three, pw_zero); |
| |
| this_1 = vec_ld(0, inptr_1); |
| this_1l = (__vector short)VEC_UNPACKHU(this_1); |
| this_1h = (__vector short)VEC_UNPACKLU(this_1); |
| thiscolsum_1l = vec_add(this0l, this_1l); |
| thiscolsum_1h = vec_add(this0h, this_1h); |
| lastcolsum_1h = thiscolsum_1h; |
| p_lastcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1l, last_index_col0); |
| p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index); |
| |
| this1 = vec_ld(0, inptr1); |
| this1l = (__vector short)VEC_UNPACKHU(this1); |
| this1h = (__vector short)VEC_UNPACKLU(this1); |
| thiscolsum1l = vec_add(this0l, this1l); |
| thiscolsum1h = vec_add(this0h, this1h); |
| lastcolsum1h = thiscolsum1h; |
| p_lastcolsum1l = vec_perm(thiscolsum1l, thiscolsum1l, last_index_col0); |
| p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index); |
| |
| for (incol = downsampled_width; incol > 0; |
| incol -= 16, inptr_1 += 16, inptr0 += 16, inptr1 += 16, |
| outptr0 += 32, outptr1 += 32) { |
| |
| if (downsampled_width - incol > 0) { |
| p_lastcolsum_1l = vec_perm(lastcolsum_1h, thiscolsum_1l, last_index); |
| p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index); |
| p_lastcolsum1l = vec_perm(lastcolsum1h, thiscolsum1l, last_index); |
| p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index); |
| lastcolsum_1h = thiscolsum_1h; lastcolsum1h = thiscolsum1h; |
| } |
| |
| if (incol <= 16) { |
| p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index); |
| p_nextcolsum_1h = vec_perm(thiscolsum_1h, thiscolsum_1h, |
| next_index_lastcol); |
| p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index); |
| p_nextcolsum1h = vec_perm(thiscolsum1h, thiscolsum1h, |
| next_index_lastcol); |
| } else { |
| this0 = vec_ld(16, inptr0); |
| this0l = (__vector short)VEC_UNPACKHU(this0); |
| this0h = (__vector short)VEC_UNPACKLU(this0); |
| this0l = vec_mladd(this0l, pw_three, pw_zero); |
| this0h = vec_mladd(this0h, pw_three, pw_zero); |
| |
| this_1 = vec_ld(16, inptr_1); |
| this_1l = (__vector short)VEC_UNPACKHU(this_1); |
| this_1h = (__vector short)VEC_UNPACKLU(this_1); |
| nextcolsum_1l = vec_add(this0l, this_1l); |
| nextcolsum_1h = vec_add(this0h, this_1h); |
| p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index); |
| p_nextcolsum_1h = vec_perm(thiscolsum_1h, nextcolsum_1l, next_index); |
| |
| this1 = vec_ld(16, inptr1); |
| this1l = (__vector short)VEC_UNPACKHU(this1); |
| this1h = (__vector short)VEC_UNPACKLU(this1); |
| nextcolsum1l = vec_add(this0l, this1l); |
| nextcolsum1h = vec_add(this0h, this1h); |
| p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index); |
| p_nextcolsum1h = vec_perm(thiscolsum1h, nextcolsum1l, next_index); |
| } |
| |
| /* Process the upper row */ |
| |
| tmpl = vec_mladd(thiscolsum_1l, pw_three, pw_zero); |
| outle = vec_add(tmpl, p_lastcolsum_1l); |
| outle = vec_add(outle, pw_eight); |
| outle = vec_sr(outle, pw_four); |
| |
| outlo = vec_add(tmpl, p_nextcolsum_1l); |
| outlo = vec_add(outlo, pw_seven); |
| outlo = vec_sr(outlo, pw_four); |
| |
| out = vec_perm((__vector unsigned char)outle, |
| (__vector unsigned char)outlo, merge_pack_index); |
| vec_st(out, 0, outptr0); |
| |
| if (incol > 8) { |
| tmph = vec_mladd(thiscolsum_1h, pw_three, pw_zero); |
| outhe = vec_add(tmph, p_lastcolsum_1h); |
| outhe = vec_add(outhe, pw_eight); |
| outhe = vec_sr(outhe, pw_four); |
| |
| outho = vec_add(tmph, p_nextcolsum_1h); |
| outho = vec_add(outho, pw_seven); |
| outho = vec_sr(outho, pw_four); |
| |
| out = vec_perm((__vector unsigned char)outhe, |
| (__vector unsigned char)outho, merge_pack_index); |
| vec_st(out, 16, outptr0); |
| } |
| |
| /* Process the lower row */ |
| |
| tmpl = vec_mladd(thiscolsum1l, pw_three, pw_zero); |
| outle = vec_add(tmpl, p_lastcolsum1l); |
| outle = vec_add(outle, pw_eight); |
| outle = vec_sr(outle, pw_four); |
| |
| outlo = vec_add(tmpl, p_nextcolsum1l); |
| outlo = vec_add(outlo, pw_seven); |
| outlo = vec_sr(outlo, pw_four); |
| |
| out = vec_perm((__vector unsigned char)outle, |
| (__vector unsigned char)outlo, merge_pack_index); |
| vec_st(out, 0, outptr1); |
| |
| if (incol > 8) { |
| tmph = vec_mladd(thiscolsum1h, pw_three, pw_zero); |
| outhe = vec_add(tmph, p_lastcolsum1h); |
| outhe = vec_add(outhe, pw_eight); |
| outhe = vec_sr(outhe, pw_four); |
| |
| outho = vec_add(tmph, p_nextcolsum1h); |
| outho = vec_add(outho, pw_seven); |
| outho = vec_sr(outho, pw_four); |
| |
| out = vec_perm((__vector unsigned char)outhe, |
| (__vector unsigned char)outho, merge_pack_index); |
| vec_st(out, 16, outptr1); |
| } |
| |
| thiscolsum_1l = nextcolsum_1l; thiscolsum_1h = nextcolsum_1h; |
| thiscolsum1l = nextcolsum1l; thiscolsum1h = nextcolsum1h; |
| } |
| } |
| } |
| |
| |
| /* These are rarely used (mainly just for decompressing YCCK images) */ |
| |
| void jsimd_h2v1_upsample_altivec(int max_v_samp_factor, |
| JDIMENSION output_width, |
| JSAMPARRAY input_data, |
| JSAMPARRAY *output_data_ptr) |
| { |
| JSAMPARRAY output_data = *output_data_ptr; |
| JSAMPROW inptr, outptr; |
| int inrow, incol; |
| |
| __vector unsigned char in, inl, inh; |
| |
| for (inrow = 0; inrow < max_v_samp_factor; inrow++) { |
| inptr = input_data[inrow]; |
| outptr = output_data[inrow]; |
| |
| for (incol = (output_width + 31) & (~31); incol > 0; |
| incol -= 64, inptr += 32, outptr += 64) { |
| |
| in = vec_ld(0, inptr); |
| inl = vec_mergeh(in, in); |
| inh = vec_mergel(in, in); |
| |
| vec_st(inl, 0, outptr); |
| vec_st(inh, 16, outptr); |
| |
| if (incol > 32) { |
| in = vec_ld(16, inptr); |
| inl = vec_mergeh(in, in); |
| inh = vec_mergel(in, in); |
| |
| vec_st(inl, 32, outptr); |
| vec_st(inh, 48, outptr); |
| } |
| } |
| } |
| } |
| |
| |
| void jsimd_h2v2_upsample_altivec(int max_v_samp_factor, |
| JDIMENSION output_width, |
| JSAMPARRAY input_data, |
| JSAMPARRAY *output_data_ptr) |
| { |
| JSAMPARRAY output_data = *output_data_ptr; |
| JSAMPROW inptr, outptr0, outptr1; |
| int inrow, outrow, incol; |
| |
| __vector unsigned char in, inl, inh; |
| |
| for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) { |
| |
| inptr = input_data[inrow]; |
| outptr0 = output_data[outrow++]; |
| outptr1 = output_data[outrow++]; |
| |
| for (incol = (output_width + 31) & (~31); incol > 0; |
| incol -= 64, inptr += 32, outptr0 += 64, outptr1 += 64) { |
| |
| in = vec_ld(0, inptr); |
| inl = vec_mergeh(in, in); |
| inh = vec_mergel(in, in); |
| |
| vec_st(inl, 0, outptr0); |
| vec_st(inl, 0, outptr1); |
| |
| vec_st(inh, 16, outptr0); |
| vec_st(inh, 16, outptr1); |
| |
| if (incol > 32) { |
| in = vec_ld(16, inptr); |
| inl = vec_mergeh(in, in); |
| inh = vec_mergel(in, in); |
| |
| vec_st(inl, 32, outptr0); |
| vec_st(inl, 32, outptr1); |
| |
| vec_st(inh, 48, outptr0); |
| vec_st(inh, 48, outptr1); |
| } |
| } |
| } |
| } |
