;
; jdsample.asm - upsampling (SSE2)
;
; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
;
; Based on the x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler),
; can *not* be assembled with Microsoft's MASM or any compatible
; assembler (including Borland's Turbo Assembler).
; NASM is available from http://nasm.sourceforge.net/ or
; http://sourceforge.net/project/showfiles.php?group_id=6208
;
; [TAB8]

%include "jsimdext.inc"

; --------------------------------------------------------------------------
        SECTION SEG_CONST

        alignz  16
        global  EXTN(jconst_fancy_upsample_sse2) PRIVATE

EXTN(jconst_fancy_upsample_sse2):

PW_ONE          times 8 dw  1
PW_TWO          times 8 dw  2
PW_THREE        times 8 dw  3
PW_SEVEN        times 8 dw  7
PW_EIGHT        times 8 dw  8

        alignz  16

; --------------------------------------------------------------------------
        SECTION SEG_TEXT
        BITS    32
;
; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
;
; The upsampling algorithm is linear interpolation between pixel centers,
; also known as a "triangle filter".  This is a good compromise between
; speed and visual quality.  The centers of the output pixels are 1/4 and 3/4
; of the way between input pixel centers.
;
; GLOBAL(void)
; jsimd_h2v1_fancy_upsample_sse2 (int max_v_samp_factor,
;                                 JDIMENSION downsampled_width,
;                                 JSAMPARRAY input_data,
;                                 JSAMPARRAY *output_data_ptr);
;

%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
%define downsamp_width(b)       (b)+12          ; JDIMENSION downsampled_width
%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr

        align   16
        global  EXTN(jsimd_h2v1_fancy_upsample_sse2) PRIVATE

EXTN(jsimd_h2v1_fancy_upsample_sse2):
        push    ebp
        mov     ebp,esp
        pushpic ebx
;       push    ecx             ; need not be preserved
;       push    edx             ; need not be preserved
        push    esi
        push    edi

        get_GOT ebx             ; get GOT address

        mov     eax, JDIMENSION [downsamp_width(ebp)]  ; colctr
        test    eax,eax
        jz      near .return

        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
        test    ecx,ecx
        jz      near .return

        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
        mov     edi, POINTER [output_data_ptr(ebp)]
        mov     edi, JSAMPARRAY [edi]                   ; output_data
        alignx  16,7
.rowloop:
        push    eax                     ; colctr
        push    edi
        push    esi

        mov     esi, JSAMPROW [esi]     ; inptr
        mov     edi, JSAMPROW [edi]     ; outptr

        test    eax, SIZEOF_XMMWORD-1
        jz      short .skip
        mov     dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
        mov     JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl    ; insert a dummy sample
.skip:
        pxor    xmm0,xmm0               ; xmm0=(all 0's)
        pcmpeqb xmm7,xmm7
        psrldq  xmm7,(SIZEOF_XMMWORD-1)
        pand    xmm7, XMMWORD [esi+0*SIZEOF_XMMWORD]

        add     eax, byte SIZEOF_XMMWORD-1
        and     eax, byte -SIZEOF_XMMWORD
        cmp     eax, byte SIZEOF_XMMWORD
        ja      short .columnloop
        alignx  16,7

.columnloop_last:
        pcmpeqb xmm6,xmm6
        pslldq  xmm6,(SIZEOF_XMMWORD-1)
        pand    xmm6, XMMWORD [esi+0*SIZEOF_XMMWORD]
        jmp     short .upsample
        alignx  16,7

.columnloop:
        movdqa  xmm6, XMMWORD [esi+1*SIZEOF_XMMWORD]
        pslldq  xmm6,(SIZEOF_XMMWORD-1)

.upsample:
        movdqa  xmm1, XMMWORD [esi+0*SIZEOF_XMMWORD]
        movdqa  xmm2,xmm1
        movdqa  xmm3,xmm1               ; xmm1=( 0  1  2 ... 13 14 15)
        pslldq  xmm2,1                  ; xmm2=(--  0  1 ... 12 13 14)
        psrldq  xmm3,1                  ; xmm3=( 1  2  3 ... 14 15 --)

        por     xmm2,xmm7               ; xmm2=(-1  0  1 ... 12 13 14)
        por     xmm3,xmm6               ; xmm3=( 1  2  3 ... 14 15 16)

        movdqa  xmm7,xmm1
        psrldq  xmm7,(SIZEOF_XMMWORD-1) ; xmm7=(15 -- -- ... -- -- --)

        movdqa    xmm4,xmm1
        punpcklbw xmm1,xmm0             ; xmm1=( 0  1  2  3  4  5  6  7)
        punpckhbw xmm4,xmm0             ; xmm4=( 8  9 10 11 12 13 14 15)
        movdqa    xmm5,xmm2
        punpcklbw xmm2,xmm0             ; xmm2=(-1  0  1  2  3  4  5  6)
        punpckhbw xmm5,xmm0             ; xmm5=( 7  8  9 10 11 12 13 14)
        movdqa    xmm6,xmm3
        punpcklbw xmm3,xmm0             ; xmm3=( 1  2  3  4  5  6  7  8)
        punpckhbw xmm6,xmm0             ; xmm6=( 9 10 11 12 13 14 15 16)

        pmullw  xmm1,[GOTOFF(ebx,PW_THREE)]
        pmullw  xmm4,[GOTOFF(ebx,PW_THREE)]
        paddw   xmm2,[GOTOFF(ebx,PW_ONE)]
        paddw   xmm5,[GOTOFF(ebx,PW_ONE)]
        paddw   xmm3,[GOTOFF(ebx,PW_TWO)]
        paddw   xmm6,[GOTOFF(ebx,PW_TWO)]

        paddw   xmm2,xmm1
        paddw   xmm5,xmm4
        psrlw   xmm2,2                  ; xmm2=OutLE=( 0  2  4  6  8 10 12 14)
        psrlw   xmm5,2                  ; xmm5=OutHE=(16 18 20 22 24 26 28 30)
        paddw   xmm3,xmm1
        paddw   xmm6,xmm4
        psrlw   xmm3,2                  ; xmm3=OutLO=( 1  3  5  7  9 11 13 15)
        psrlw   xmm6,2                  ; xmm6=OutHO=(17 19 21 23 25 27 29 31)

        psllw   xmm3,BYTE_BIT
        psllw   xmm6,BYTE_BIT
        por     xmm2,xmm3               ; xmm2=OutL=( 0  1  2 ... 13 14 15)
        por     xmm5,xmm6               ; xmm5=OutH=(16 17 18 ... 29 30 31)

        movdqa  XMMWORD [edi+0*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [edi+1*SIZEOF_XMMWORD], xmm5

        sub     eax, byte SIZEOF_XMMWORD
        add     esi, byte 1*SIZEOF_XMMWORD      ; inptr
        add     edi, byte 2*SIZEOF_XMMWORD      ; outptr
        cmp     eax, byte SIZEOF_XMMWORD
        ja      near .columnloop
        test    eax,eax
        jnz     near .columnloop_last

        pop     esi
        pop     edi
        pop     eax

        add     esi, byte SIZEOF_JSAMPROW       ; input_data
        add     edi, byte SIZEOF_JSAMPROW       ; output_data
        dec     ecx                             ; rowctr
        jg      near .rowloop

.return:
        pop     edi
        pop     esi
;       pop     edx             ; need not be preserved
;       pop     ecx             ; need not be preserved
        poppic  ebx
        pop     ebp
        ret

; --------------------------------------------------------------------------
;
; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
; Again a triangle filter; see comments for h2v1 case, above.
;
; GLOBAL(void)
; jsimd_h2v2_fancy_upsample_sse2 (int max_v_samp_factor,
;                                 JDIMENSION downsampled_width,
;                                 JSAMPARRAY input_data,
;                                 JSAMPARRAY *output_data_ptr);
;

%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
%define downsamp_width(b)       (b)+12          ; JDIMENSION downsampled_width
%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr

%define original_ebp    ebp+0
%define wk(i)           ebp-(WK_NUM-(i))*SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
%define WK_NUM          4
%define gotptr          wk(0)-SIZEOF_POINTER    ; void *gotptr

        align   16
        global  EXTN(jsimd_h2v2_fancy_upsample_sse2) PRIVATE

EXTN(jsimd_h2v2_fancy_upsample_sse2):
        push    ebp
        mov     eax,esp                         ; eax = original ebp
        sub     esp, byte 4
        and     esp, byte (-SIZEOF_XMMWORD)     ; align to 128 bits
        mov     [esp],eax
        mov     ebp,esp                         ; ebp = aligned ebp
        lea     esp, [wk(0)]
        pushpic eax             ; make a room for GOT address
        push    ebx
;       push    ecx             ; need not be preserved
;       push    edx             ; need not be preserved
        push    esi
        push    edi

        get_GOT ebx                     ; get GOT address
        movpic  POINTER [gotptr], ebx   ; save GOT address

        mov     edx,eax                         ; edx = original ebp
        mov     eax, JDIMENSION [downsamp_width(edx)]  ; colctr
        test    eax,eax
        jz      near .return

        mov     ecx, INT [max_v_samp(edx)]      ; rowctr
        test    ecx,ecx
        jz      near .return

        mov     esi, JSAMPARRAY [input_data(edx)]       ; input_data
        mov     edi, POINTER [output_data_ptr(edx)]
        mov     edi, JSAMPARRAY [edi]                   ; output_data
        alignx  16,7
.rowloop:
        push    eax                                     ; colctr
        push    ecx
        push    edi
        push    esi

        mov     ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]   ; inptr1(above)
        mov     ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]   ; inptr0
        mov     esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]   ; inptr1(below)
        mov     edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]   ; outptr0
        mov     edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]   ; outptr1

        test    eax, SIZEOF_XMMWORD-1
        jz      short .skip
        push    edx
        mov     dl, JSAMPLE [ecx+(eax-1)*SIZEOF_JSAMPLE]
        mov     JSAMPLE [ecx+eax*SIZEOF_JSAMPLE], dl
        mov     dl, JSAMPLE [ebx+(eax-1)*SIZEOF_JSAMPLE]
        mov     JSAMPLE [ebx+eax*SIZEOF_JSAMPLE], dl
        mov     dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
        mov     JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl    ; insert a dummy sample
        pop     edx
.skip:
        ; -- process the first column block

        movdqa  xmm0, XMMWORD [ebx+0*SIZEOF_XMMWORD]    ; xmm0=row[ 0][0]
        movdqa  xmm1, XMMWORD [ecx+0*SIZEOF_XMMWORD]    ; xmm1=row[-1][0]
        movdqa  xmm2, XMMWORD [esi+0*SIZEOF_XMMWORD]    ; xmm2=row[+1][0]

        pushpic ebx
        movpic  ebx, POINTER [gotptr]   ; load GOT address

        pxor      xmm3,xmm3             ; xmm3=(all 0's)
        movdqa    xmm4,xmm0
        punpcklbw xmm0,xmm3             ; xmm0=row[ 0]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm4,xmm3             ; xmm4=row[ 0]( 8  9 10 11 12 13 14 15)
        movdqa    xmm5,xmm1
        punpcklbw xmm1,xmm3             ; xmm1=row[-1]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm5,xmm3             ; xmm5=row[-1]( 8  9 10 11 12 13 14 15)
        movdqa    xmm6,xmm2
        punpcklbw xmm2,xmm3             ; xmm2=row[+1]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm6,xmm3             ; xmm6=row[+1]( 8  9 10 11 12 13 14 15)

        pmullw  xmm0,[GOTOFF(ebx,PW_THREE)]
        pmullw  xmm4,[GOTOFF(ebx,PW_THREE)]

        pcmpeqb xmm7,xmm7
        psrldq  xmm7,(SIZEOF_XMMWORD-2)

        paddw   xmm1,xmm0               ; xmm1=Int0L=( 0  1  2  3  4  5  6  7)
        paddw   xmm5,xmm4               ; xmm5=Int0H=( 8  9 10 11 12 13 14 15)
        paddw   xmm2,xmm0               ; xmm2=Int1L=( 0  1  2  3  4  5  6  7)
        paddw   xmm6,xmm4               ; xmm6=Int1H=( 8  9 10 11 12 13 14 15)

        movdqa  XMMWORD [edx+0*SIZEOF_XMMWORD], xmm1    ; temporarily save
        movdqa  XMMWORD [edx+1*SIZEOF_XMMWORD], xmm5    ; the intermediate data
        movdqa  XMMWORD [edi+0*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [edi+1*SIZEOF_XMMWORD], xmm6

        pand    xmm1,xmm7               ; xmm1=( 0 -- -- -- -- -- -- --)
        pand    xmm2,xmm7               ; xmm2=( 0 -- -- -- -- -- -- --)

        movdqa  XMMWORD [wk(0)], xmm1
        movdqa  XMMWORD [wk(1)], xmm2

        poppic  ebx

        add     eax, byte SIZEOF_XMMWORD-1
        and     eax, byte -SIZEOF_XMMWORD
        cmp     eax, byte SIZEOF_XMMWORD
        ja      short .columnloop
        alignx  16,7

.columnloop_last:
        ; -- process the last column block

        pushpic ebx
        movpic  ebx, POINTER [gotptr]   ; load GOT address

        pcmpeqb xmm1,xmm1
        pslldq  xmm1,(SIZEOF_XMMWORD-2)
        movdqa  xmm2,xmm1

        pand    xmm1, XMMWORD [edx+1*SIZEOF_XMMWORD]
        pand    xmm2, XMMWORD [edi+1*SIZEOF_XMMWORD]

        movdqa  XMMWORD [wk(2)], xmm1   ; xmm1=(-- -- -- -- -- -- -- 15)
        movdqa  XMMWORD [wk(3)], xmm2   ; xmm2=(-- -- -- -- -- -- -- 15)

        jmp     near .upsample
        alignx  16,7

.columnloop:
        ; -- process the next column block

        movdqa  xmm0, XMMWORD [ebx+1*SIZEOF_XMMWORD]    ; xmm0=row[ 0][1]
        movdqa  xmm1, XMMWORD [ecx+1*SIZEOF_XMMWORD]    ; xmm1=row[-1][1]
        movdqa  xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]    ; xmm2=row[+1][1]

        pushpic ebx
        movpic  ebx, POINTER [gotptr]   ; load GOT address

        pxor      xmm3,xmm3             ; xmm3=(all 0's)
        movdqa    xmm4,xmm0
        punpcklbw xmm0,xmm3             ; xmm0=row[ 0]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm4,xmm3             ; xmm4=row[ 0]( 8  9 10 11 12 13 14 15)
        movdqa    xmm5,xmm1
        punpcklbw xmm1,xmm3             ; xmm1=row[-1]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm5,xmm3             ; xmm5=row[-1]( 8  9 10 11 12 13 14 15)
        movdqa    xmm6,xmm2
        punpcklbw xmm2,xmm3             ; xmm2=row[+1]( 0  1  2  3  4  5  6  7)
        punpckhbw xmm6,xmm3             ; xmm6=row[+1]( 8  9 10 11 12 13 14 15)

        pmullw  xmm0,[GOTOFF(ebx,PW_THREE)]
        pmullw  xmm4,[GOTOFF(ebx,PW_THREE)]

        paddw   xmm1,xmm0               ; xmm1=Int0L=( 0  1  2  3  4  5  6  7)
        paddw   xmm5,xmm4               ; xmm5=Int0H=( 8  9 10 11 12 13 14 15)
        paddw   xmm2,xmm0               ; xmm2=Int1L=( 0  1  2  3  4  5  6  7)
        paddw   xmm6,xmm4               ; xmm6=Int1H=( 8  9 10 11 12 13 14 15)

        movdqa  XMMWORD [edx+2*SIZEOF_XMMWORD], xmm1    ; temporarily save
        movdqa  XMMWORD [edx+3*SIZEOF_XMMWORD], xmm5    ; the intermediate data
        movdqa  XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [edi+3*SIZEOF_XMMWORD], xmm6

        pslldq  xmm1,(SIZEOF_XMMWORD-2) ; xmm1=(-- -- -- -- -- -- --  0)
        pslldq  xmm2,(SIZEOF_XMMWORD-2) ; xmm2=(-- -- -- -- -- -- --  0)

        movdqa  XMMWORD [wk(2)], xmm1
        movdqa  XMMWORD [wk(3)], xmm2

.upsample:
        ; -- process the upper row

        movdqa  xmm7, XMMWORD [edx+0*SIZEOF_XMMWORD]
        movdqa  xmm3, XMMWORD [edx+1*SIZEOF_XMMWORD]

        movdqa  xmm0,xmm7               ; xmm7=Int0L=( 0  1  2  3  4  5  6  7)
        movdqa  xmm4,xmm3               ; xmm3=Int0H=( 8  9 10 11 12 13 14 15)
        psrldq  xmm0,2                  ; xmm0=( 1  2  3  4  5  6  7 --)
        pslldq  xmm4,(SIZEOF_XMMWORD-2) ; xmm4=(-- -- -- -- -- -- --  8)
        movdqa  xmm5,xmm7
        movdqa  xmm6,xmm3
        psrldq  xmm5,(SIZEOF_XMMWORD-2) ; xmm5=( 7 -- -- -- -- -- -- --)
        pslldq  xmm6,2                  ; xmm6=(--  8  9 10 11 12 13 14)

        por     xmm0,xmm4               ; xmm0=( 1  2  3  4  5  6  7  8)
        por     xmm5,xmm6               ; xmm5=( 7  8  9 10 11 12 13 14)

        movdqa  xmm1,xmm7
        movdqa  xmm2,xmm3
        pslldq  xmm1,2                  ; xmm1=(--  0  1  2  3  4  5  6)
        psrldq  xmm2,2                  ; xmm2=( 9 10 11 12 13 14 15 --)
        movdqa  xmm4,xmm3
        psrldq  xmm4,(SIZEOF_XMMWORD-2) ; xmm4=(15 -- -- -- -- -- -- --)

        por     xmm1, XMMWORD [wk(0)]   ; xmm1=(-1  0  1  2  3  4  5  6)
        por     xmm2, XMMWORD [wk(2)]   ; xmm2=( 9 10 11 12 13 14 15 16)

        movdqa  XMMWORD [wk(0)], xmm4

        pmullw  xmm7,[GOTOFF(ebx,PW_THREE)]
        pmullw  xmm3,[GOTOFF(ebx,PW_THREE)]
        paddw   xmm1,[GOTOFF(ebx,PW_EIGHT)]
        paddw   xmm5,[GOTOFF(ebx,PW_EIGHT)]
        paddw   xmm0,[GOTOFF(ebx,PW_SEVEN)]
        paddw   xmm2,[GOTOFF(ebx,PW_SEVEN)]

        paddw   xmm1,xmm7
        paddw   xmm5,xmm3
        psrlw   xmm1,4                  ; xmm1=Out0LE=( 0  2  4  6  8 10 12 14)
        psrlw   xmm5,4                  ; xmm5=Out0HE=(16 18 20 22 24 26 28 30)
        paddw   xmm0,xmm7
        paddw   xmm2,xmm3
        psrlw   xmm0,4                  ; xmm0=Out0LO=( 1  3  5  7  9 11 13 15)
        psrlw   xmm2,4                  ; xmm2=Out0HO=(17 19 21 23 25 27 29 31)

        psllw   xmm0,BYTE_BIT
        psllw   xmm2,BYTE_BIT
        por     xmm1,xmm0               ; xmm1=Out0L=( 0  1  2 ... 13 14 15)
        por     xmm5,xmm2               ; xmm5=Out0H=(16 17 18 ... 29 30 31)

        movdqa  XMMWORD [edx+0*SIZEOF_XMMWORD], xmm1
        movdqa  XMMWORD [edx+1*SIZEOF_XMMWORD], xmm5

        ; -- process the lower row

        movdqa  xmm6, XMMWORD [edi+0*SIZEOF_XMMWORD]
        movdqa  xmm4, XMMWORD [edi+1*SIZEOF_XMMWORD]

        movdqa  xmm7,xmm6               ; xmm6=Int1L=( 0  1  2  3  4  5  6  7)
        movdqa  xmm3,xmm4               ; xmm4=Int1H=( 8  9 10 11 12 13 14 15)
        psrldq  xmm7,2                  ; xmm7=( 1  2  3  4  5  6  7 --)
        pslldq  xmm3,(SIZEOF_XMMWORD-2) ; xmm3=(-- -- -- -- -- -- --  8)
        movdqa  xmm0,xmm6
        movdqa  xmm2,xmm4
        psrldq  xmm0,(SIZEOF_XMMWORD-2) ; xmm0=( 7 -- -- -- -- -- -- --)
        pslldq  xmm2,2                  ; xmm2=(--  8  9 10 11 12 13 14)

        por     xmm7,xmm3               ; xmm7=( 1  2  3  4  5  6  7  8)
        por     xmm0,xmm2               ; xmm0=( 7  8  9 10 11 12 13 14)

        movdqa  xmm1,xmm6
        movdqa  xmm5,xmm4
        pslldq  xmm1,2                  ; xmm1=(--  0  1  2  3  4  5  6)
        psrldq  xmm5,2                  ; xmm5=( 9 10 11 12 13 14 15 --)
        movdqa  xmm3,xmm4
        psrldq  xmm3,(SIZEOF_XMMWORD-2) ; xmm3=(15 -- -- -- -- -- -- --)

        por     xmm1, XMMWORD [wk(1)]   ; xmm1=(-1  0  1  2  3  4  5  6)
        por     xmm5, XMMWORD [wk(3)]   ; xmm5=( 9 10 11 12 13 14 15 16)

        movdqa  XMMWORD [wk(1)], xmm3

        pmullw  xmm6,[GOTOFF(ebx,PW_THREE)]
        pmullw  xmm4,[GOTOFF(ebx,PW_THREE)]
        paddw   xmm1,[GOTOFF(ebx,PW_EIGHT)]
        paddw   xmm0,[GOTOFF(ebx,PW_EIGHT)]
        paddw   xmm7,[GOTOFF(ebx,PW_SEVEN)]
        paddw   xmm5,[GOTOFF(ebx,PW_SEVEN)]

        paddw   xmm1,xmm6
        paddw   xmm0,xmm4
        psrlw   xmm1,4                  ; xmm1=Out1LE=( 0  2  4  6  8 10 12 14)
        psrlw   xmm0,4                  ; xmm0=Out1HE=(16 18 20 22 24 26 28 30)
        paddw   xmm7,xmm6
        paddw   xmm5,xmm4
        psrlw   xmm7,4                  ; xmm7=Out1LO=( 1  3  5  7  9 11 13 15)
        psrlw   xmm5,4                  ; xmm5=Out1HO=(17 19 21 23 25 27 29 31)

        psllw   xmm7,BYTE_BIT
        psllw   xmm5,BYTE_BIT
        por     xmm1,xmm7               ; xmm1=Out1L=( 0  1  2 ... 13 14 15)
        por     xmm0,xmm5               ; xmm0=Out1H=(16 17 18 ... 29 30 31)

        movdqa  XMMWORD [edi+0*SIZEOF_XMMWORD], xmm1
        movdqa  XMMWORD [edi+1*SIZEOF_XMMWORD], xmm0

        poppic  ebx

        sub     eax, byte SIZEOF_XMMWORD
        add     ecx, byte 1*SIZEOF_XMMWORD      ; inptr1(above)
        add     ebx, byte 1*SIZEOF_XMMWORD      ; inptr0
        add     esi, byte 1*SIZEOF_XMMWORD      ; inptr1(below)
        add     edx, byte 2*SIZEOF_XMMWORD      ; outptr0
        add     edi, byte 2*SIZEOF_XMMWORD      ; outptr1
        cmp     eax, byte SIZEOF_XMMWORD
        ja      near .columnloop
        test    eax,eax
        jnz     near .columnloop_last

        pop     esi
        pop     edi
        pop     ecx
        pop     eax

        add     esi, byte 1*SIZEOF_JSAMPROW     ; input_data
        add     edi, byte 2*SIZEOF_JSAMPROW     ; output_data
        sub     ecx, byte 2                     ; rowctr
        jg      near .rowloop

.return:
        pop     edi
        pop     esi
;       pop     edx             ; need not be preserved
;       pop     ecx             ; need not be preserved
        pop     ebx
        mov     esp,ebp         ; esp <- aligned ebp
        pop     esp             ; esp <- original ebp
        pop     ebp
        ret

; --------------------------------------------------------------------------
;
; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jsimd_h2v1_upsample_sse2 (int max_v_samp_factor,
;                           JDIMENSION output_width,
;                           JSAMPARRAY input_data,
;                           JSAMPARRAY *output_data_ptr);
;

%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
%define output_width(b)         (b)+12          ; JDIMENSION output_width
%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr

        align   16
        global  EXTN(jsimd_h2v1_upsample_sse2) PRIVATE

EXTN(jsimd_h2v1_upsample_sse2):
        push    ebp
        mov     ebp,esp
;       push    ebx             ; unused
;       push    ecx             ; need not be preserved
;       push    edx             ; need not be preserved
        push    esi
        push    edi

        mov     edx, JDIMENSION [output_width(ebp)]
        add     edx, byte (2*SIZEOF_XMMWORD)-1
        and     edx, byte -(2*SIZEOF_XMMWORD)
        jz      short .return

        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
        test    ecx,ecx
        jz      short .return

        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
        mov     edi, POINTER [output_data_ptr(ebp)]
        mov     edi, JSAMPARRAY [edi]                   ; output_data
        alignx  16,7
.rowloop:
        push    edi
        push    esi

        mov     esi, JSAMPROW [esi]             ; inptr
        mov     edi, JSAMPROW [edi]             ; outptr
        mov     eax,edx                         ; colctr
        alignx  16,7
.columnloop:

        movdqa  xmm0, XMMWORD [esi+0*SIZEOF_XMMWORD]

        movdqa    xmm1,xmm0
        punpcklbw xmm0,xmm0
        punpckhbw xmm1,xmm1

        movdqa  XMMWORD [edi+0*SIZEOF_XMMWORD], xmm0
        movdqa  XMMWORD [edi+1*SIZEOF_XMMWORD], xmm1

        sub     eax, byte 2*SIZEOF_XMMWORD
        jz      short .nextrow

        movdqa  xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]

        movdqa    xmm3,xmm2
        punpcklbw xmm2,xmm2
        punpckhbw xmm3,xmm3

        movdqa  XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [edi+3*SIZEOF_XMMWORD], xmm3

        sub     eax, byte 2*SIZEOF_XMMWORD
        jz      short .nextrow

        add     esi, byte 2*SIZEOF_XMMWORD      ; inptr
        add     edi, byte 4*SIZEOF_XMMWORD      ; outptr
        jmp     short .columnloop
        alignx  16,7

.nextrow:
        pop     esi
        pop     edi

        add     esi, byte SIZEOF_JSAMPROW       ; input_data
        add     edi, byte SIZEOF_JSAMPROW       ; output_data
        dec     ecx                             ; rowctr
        jg      short .rowloop

.return:
        pop     edi
        pop     esi
;       pop     edx             ; need not be preserved
;       pop     ecx             ; need not be preserved
;       pop     ebx             ; unused
        pop     ebp
        ret

; --------------------------------------------------------------------------
;
; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jsimd_h2v2_upsample_sse2 (nt max_v_samp_factor,
;                           JDIMENSION output_width,
;                           JSAMPARRAY input_data,
;                           JSAMPARRAY *output_data_ptr);
;

%define max_v_samp(b)           (b)+8           ; int max_v_samp_factor
%define output_width(b)         (b)+12          ; JDIMENSION output_width
%define input_data(b)           (b)+16          ; JSAMPARRAY input_data
%define output_data_ptr(b)      (b)+20          ; JSAMPARRAY *output_data_ptr

        align   16
        global  EXTN(jsimd_h2v2_upsample_sse2) PRIVATE

EXTN(jsimd_h2v2_upsample_sse2):
        push    ebp
        mov     ebp,esp
        push    ebx
;       push    ecx             ; need not be preserved
;       push    edx             ; need not be preserved
        push    esi
        push    edi

        mov     edx, JDIMENSION [output_width(ebp)]
        add     edx, byte (2*SIZEOF_XMMWORD)-1
        and     edx, byte -(2*SIZEOF_XMMWORD)
        jz      near .return

        mov     ecx, INT [max_v_samp(ebp)]      ; rowctr
        test    ecx,ecx
        jz      near .return

        mov     esi, JSAMPARRAY [input_data(ebp)]       ; input_data
        mov     edi, POINTER [output_data_ptr(ebp)]
        mov     edi, JSAMPARRAY [edi]                   ; output_data
        alignx  16,7
.rowloop:
        push    edi
        push    esi

        mov     esi, JSAMPROW [esi]                     ; inptr
        mov     ebx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]   ; outptr0
        mov     edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]   ; outptr1
        mov     eax,edx                                 ; colctr
        alignx  16,7
.columnloop:

        movdqa  xmm0, XMMWORD [esi+0*SIZEOF_XMMWORD]

        movdqa    xmm1,xmm0
        punpcklbw xmm0,xmm0
        punpckhbw xmm1,xmm1

        movdqa  XMMWORD [ebx+0*SIZEOF_XMMWORD], xmm0
        movdqa  XMMWORD [ebx+1*SIZEOF_XMMWORD], xmm1
        movdqa  XMMWORD [edi+0*SIZEOF_XMMWORD], xmm0
        movdqa  XMMWORD [edi+1*SIZEOF_XMMWORD], xmm1

        sub     eax, byte 2*SIZEOF_XMMWORD
        jz      short .nextrow

        movdqa  xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]

        movdqa    xmm3,xmm2
        punpcklbw xmm2,xmm2
        punpckhbw xmm3,xmm3

        movdqa  XMMWORD [ebx+2*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [ebx+3*SIZEOF_XMMWORD], xmm3
        movdqa  XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
        movdqa  XMMWORD [edi+3*SIZEOF_XMMWORD], xmm3

        sub     eax, byte 2*SIZEOF_XMMWORD
        jz      short .nextrow

        add     esi, byte 2*SIZEOF_XMMWORD      ; inptr
        add     ebx, byte 4*SIZEOF_XMMWORD      ; outptr0
        add     edi, byte 4*SIZEOF_XMMWORD      ; outptr1
        jmp     short .columnloop
        alignx  16,7

.nextrow:
        pop     esi
        pop     edi

        add     esi, byte 1*SIZEOF_JSAMPROW     ; input_data
        add     edi, byte 2*SIZEOF_JSAMPROW     ; output_data
        sub     ecx, byte 2                     ; rowctr
        jg      short .rowloop

.return:
        pop     edi
        pop     esi
;       pop     edx             ; need not be preserved
;       pop     ecx             ; need not be preserved
        pop     ebx
        pop     ebp
        ret

; For some reason, the OS X linker does not honor the request to align the
; segment unless we do this.
        align   16