;
; jdsammmx.asm - upsampling (MMX)
;
; x86 SIMD extension for IJG JPEG library
; Copyright (C) 1999-2006, MIYASAKA Masaru.
; For conditions of distribution and use, see copyright notice in jsimdext.inc
;
; This file should be assembled with NASM (Netwide Assembler),
; can *not* be assembled with Microsoft's MASM or any compatible
; assembler (including Borland's Turbo Assembler).
; NASM is available from http://nasm.sourceforge.net/ or
; http://sourceforge.net/project/showfiles.php?group_id=6208
;
; Last Modified : February 4, 2006
;
; [TAB8]

%include "jsimdext.inc"
%include "jcolsamp.inc"

%ifdef JDSAMPLE_FANCY_MMX_SUPPORTED

; --------------------------------------------------------------------------
	SECTION	SEG_CONST

	alignz	16
	global	EXTN(jconst_fancy_upsample_mmx)

EXTN(jconst_fancy_upsample_mmx):

PW_ONE		times 4 dw  1
PW_TWO		times 4 dw  2
PW_THREE	times 4 dw  3
PW_SEVEN	times 4 dw  7
PW_EIGHT	times 4 dw  8

	alignz	16

; --------------------------------------------------------------------------
	SECTION	SEG_TEXT
	BITS	32
;
; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
;
; The upsampling algorithm is linear interpolation between pixel centers,
; also known as a "triangle filter".  This is a good compromise between
; speed and visual quality.  The centers of the output pixels are 1/4 and 3/4
; of the way between input pixel centers.
;
; GLOBAL(void)
; jpeg_h2v1_fancy_upsample_mmx (j_decompress_ptr cinfo,
;                               jpeg_component_info * compptr,
;                               JSAMPARRAY input_data,
;                               JSAMPARRAY * output_data_ptr);
;

%define cinfo(b)		(b)+8		; j_decompress_ptr cinfo
%define compptr(b)		(b)+12		; jpeg_component_info * compptr
%define input_data(b)		(b)+16		; JSAMPARRAY input_data
%define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr

	align	16
	global	EXTN(jpeg_h2v1_fancy_upsample_mmx)

EXTN(jpeg_h2v1_fancy_upsample_mmx):
	push	ebp
	mov	ebp,esp
	pushpic	ebx
;	push	ecx		; need not be preserved
;	push	edx		; need not be preserved
	push	esi
	push	edi

	get_GOT	ebx		; get GOT address

	mov	eax, POINTER [compptr(ebp)]
	mov	eax, JDIMENSION [jcompinfo_downsampled_width(eax)]  ; colctr
	test	eax,eax
	jz	near .return

	mov	ecx, POINTER [cinfo(ebp)]
	mov	ecx, INT [jdstruct_max_v_samp_factor(ecx)]	; rowctr
	test	ecx,ecx
	jz	near .return

	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
	mov	edi, POINTER [output_data_ptr(ebp)]
	mov	edi, JSAMPARRAY [edi]			; output_data
	alignx	16,7
.rowloop:
	push	eax			; colctr
	push	edi
	push	esi

	mov	esi, JSAMPROW [esi]	; inptr
	mov	edi, JSAMPROW [edi]	; outptr

	test	eax, SIZEOF_MMWORD-1
	jz	short .skip
	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
.skip:
	pxor	mm0,mm0			; mm0=(all 0's)
	pcmpeqb	mm7,mm7
	psrlq	mm7,(SIZEOF_MMWORD-1)*BYTE_BIT
	pand	mm7, MMWORD [esi+0*SIZEOF_MMWORD]

	add	eax, byte SIZEOF_MMWORD-1
	and	eax, byte -SIZEOF_MMWORD
	cmp	eax, byte SIZEOF_MMWORD
	ja	short .columnloop
	alignx	16,7

.columnloop_last:
	pcmpeqb	mm6,mm6
	psllq	mm6,(SIZEOF_MMWORD-1)*BYTE_BIT
	pand	mm6, MMWORD [esi+0*SIZEOF_MMWORD]
	jmp	short .upsample
	alignx	16,7

.columnloop:
	movq	mm6, MMWORD [esi+1*SIZEOF_MMWORD]
	psllq	mm6,(SIZEOF_MMWORD-1)*BYTE_BIT

.upsample:
	movq	mm1, MMWORD [esi+0*SIZEOF_MMWORD]
	movq	mm2,mm1
	movq	mm3,mm1			; mm1=( 0 1 2 3 4 5 6 7)
	psllq	mm2,BYTE_BIT		; mm2=( - 0 1 2 3 4 5 6)
	psrlq	mm3,BYTE_BIT		; mm3=( 1 2 3 4 5 6 7 -)

	por	mm2,mm7			; mm2=(-1 0 1 2 3 4 5 6)
	por	mm3,mm6			; mm3=( 1 2 3 4 5 6 7 8)

	movq	mm7,mm1
	psrlq	mm7,(SIZEOF_MMWORD-1)*BYTE_BIT	; mm7=( 7 - - - - - - -)

	movq      mm4,mm1
	punpcklbw mm1,mm0		; mm1=( 0 1 2 3)
	punpckhbw mm4,mm0		; mm4=( 4 5 6 7)
	movq      mm5,mm2
	punpcklbw mm2,mm0		; mm2=(-1 0 1 2)
	punpckhbw mm5,mm0		; mm5=( 3 4 5 6)
	movq      mm6,mm3
	punpcklbw mm3,mm0		; mm3=( 1 2 3 4)
	punpckhbw mm6,mm0		; mm6=( 5 6 7 8)

	pmullw	mm1,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
	paddw	mm2,[GOTOFF(ebx,PW_ONE)]
	paddw	mm5,[GOTOFF(ebx,PW_ONE)]
	paddw	mm3,[GOTOFF(ebx,PW_TWO)]
	paddw	mm6,[GOTOFF(ebx,PW_TWO)]

	paddw	mm2,mm1
	paddw	mm5,mm4
	psrlw	mm2,2			; mm2=OutLE=( 0  2  4  6)
	psrlw	mm5,2			; mm5=OutHE=( 8 10 12 14)
	paddw	mm3,mm1
	paddw	mm6,mm4
	psrlw	mm3,2			; mm3=OutLO=( 1  3  5  7)
	psrlw	mm6,2			; mm6=OutHO=( 9 11 13 15)

	psllw	mm3,BYTE_BIT
	psllw	mm6,BYTE_BIT
	por	mm2,mm3			; mm2=OutL=( 0  1  2  3  4  5  6  7)
	por	mm5,mm6			; mm5=OutH=( 8  9 10 11 12 13 14 15)

	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm2
	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm5

	sub	eax, byte SIZEOF_MMWORD
	add	esi, byte 1*SIZEOF_MMWORD	; inptr
	add	edi, byte 2*SIZEOF_MMWORD	; outptr
	cmp	eax, byte SIZEOF_MMWORD
	ja	near .columnloop
	test	eax,eax
	jnz	near .columnloop_last

	pop	esi
	pop	edi
	pop	eax

	add	esi, byte SIZEOF_JSAMPROW	; input_data
	add	edi, byte SIZEOF_JSAMPROW	; output_data
	dec	ecx				; rowctr
	jg	near .rowloop

	emms		; empty MMX state

.return:
	pop	edi
	pop	esi
;	pop	edx		; need not be preserved
;	pop	ecx		; need not be preserved
	poppic	ebx
	pop	ebp
	ret

; --------------------------------------------------------------------------
;
; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
; Again a triangle filter; see comments for h2v1 case, above.
;
; GLOBAL(void)
; jpeg_h2v2_fancy_upsample_mmx (j_decompress_ptr cinfo,
;                               jpeg_component_info * compptr,
;                               JSAMPARRAY input_data,
;                               JSAMPARRAY * output_data_ptr);
;

%define cinfo(b)		(b)+8		; j_decompress_ptr cinfo
%define compptr(b)		(b)+12		; jpeg_component_info * compptr
%define input_data(b)		(b)+16		; JSAMPARRAY input_data
%define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr

%define original_ebp	ebp+0
%define wk(i)		ebp-(WK_NUM-(i))*SIZEOF_MMWORD	; mmword wk[WK_NUM]
%define WK_NUM		4
%define gotptr		wk(0)-SIZEOF_POINTER	; void * gotptr

	align	16
	global	EXTN(jpeg_h2v2_fancy_upsample_mmx)

EXTN(jpeg_h2v2_fancy_upsample_mmx):
	push	ebp
	mov	eax,esp				; eax = original ebp
	sub	esp, byte 4
	and	esp, byte (-SIZEOF_MMWORD)	; align to 64 bits
	mov	[esp],eax
	mov	ebp,esp				; ebp = aligned ebp
	lea	esp, [wk(0)]
	pushpic	eax		; make a room for GOT address
	push	ebx
;	push	ecx		; need not be preserved
;	push	edx		; need not be preserved
	push	esi
	push	edi

	get_GOT	ebx			; get GOT address
	movpic	POINTER [gotptr], ebx	; save GOT address

	mov	edx,eax				; edx = original ebp
	mov	eax, POINTER [compptr(edx)]
	mov	eax, JDIMENSION [jcompinfo_downsampled_width(eax)]  ; colctr
	test	eax,eax
	jz	near .return

	mov	ecx, POINTER [cinfo(edx)]
	mov	ecx, INT [jdstruct_max_v_samp_factor(ecx)]	; rowctr
	test	ecx,ecx
	jz	near .return

	mov	esi, JSAMPARRAY [input_data(edx)]	; input_data
	mov	edi, POINTER [output_data_ptr(edx)]
	mov	edi, JSAMPARRAY [edi]			; output_data
	alignx	16,7
.rowloop:
	push	eax					; colctr
	push	ecx
	push	edi
	push	esi

	mov	ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]	; inptr1(above)
	mov	ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]	; inptr0
	mov	esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]	; inptr1(below)
	mov	edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1

	test	eax, SIZEOF_MMWORD-1
	jz	short .skip
	push	edx
	mov	dl, JSAMPLE [ecx+(eax-1)*SIZEOF_JSAMPLE]
	mov	JSAMPLE [ecx+eax*SIZEOF_JSAMPLE], dl
	mov	dl, JSAMPLE [ebx+(eax-1)*SIZEOF_JSAMPLE]
	mov	JSAMPLE [ebx+eax*SIZEOF_JSAMPLE], dl
	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
	pop	edx
.skip:
	; -- process the first column block

	movq	mm0, MMWORD [ebx+0*SIZEOF_MMWORD]	; mm0=row[ 0][0]
	movq	mm1, MMWORD [ecx+0*SIZEOF_MMWORD]	; mm1=row[-1][0]
	movq	mm2, MMWORD [esi+0*SIZEOF_MMWORD]	; mm2=row[+1][0]

	pushpic	ebx
	movpic	ebx, POINTER [gotptr]	; load GOT address

	pxor      mm3,mm3		; mm3=(all 0's)
	movq      mm4,mm0
	punpcklbw mm0,mm3		; mm0=row[ 0][0]( 0 1 2 3)
	punpckhbw mm4,mm3		; mm4=row[ 0][0]( 4 5 6 7)
	movq      mm5,mm1
	punpcklbw mm1,mm3		; mm1=row[-1][0]( 0 1 2 3)
	punpckhbw mm5,mm3		; mm5=row[-1][0]( 4 5 6 7)
	movq      mm6,mm2
	punpcklbw mm2,mm3		; mm2=row[+1][0]( 0 1 2 3)
	punpckhbw mm6,mm3		; mm6=row[+1][0]( 4 5 6 7)

	pmullw	mm0,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]

	pcmpeqb	mm7,mm7
	psrlq	mm7,(SIZEOF_MMWORD-2)*BYTE_BIT

	paddw	mm1,mm0			; mm1=Int0L=( 0 1 2 3)
	paddw	mm5,mm4			; mm5=Int0H=( 4 5 6 7)
	paddw	mm2,mm0			; mm2=Int1L=( 0 1 2 3)
	paddw	mm6,mm4			; mm6=Int1H=( 4 5 6 7)

	movq	MMWORD [edx+0*SIZEOF_MMWORD], mm1	; temporarily save
	movq	MMWORD [edx+1*SIZEOF_MMWORD], mm5	; the intermediate data
	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm2
	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm6

	pand	mm1,mm7			; mm1=( 0 - - -)
	pand	mm2,mm7			; mm2=( 0 - - -)

	movq	MMWORD [wk(0)], mm1
	movq	MMWORD [wk(1)], mm2

	poppic	ebx

	add	eax, byte SIZEOF_MMWORD-1
	and	eax, byte -SIZEOF_MMWORD
	cmp	eax, byte SIZEOF_MMWORD
	ja	short .columnloop
	alignx	16,7

.columnloop_last:
	; -- process the last column block

	pushpic	ebx
	movpic	ebx, POINTER [gotptr]	; load GOT address

	pcmpeqb	mm1,mm1
	psllq	mm1,(SIZEOF_MMWORD-2)*BYTE_BIT
	movq	mm2,mm1

	pand	mm1, MMWORD [edx+1*SIZEOF_MMWORD]	; mm1=( - - - 7)
	pand	mm2, MMWORD [edi+1*SIZEOF_MMWORD]	; mm2=( - - - 7)

	movq	MMWORD [wk(2)], mm1
	movq	MMWORD [wk(3)], mm2

	jmp	short .upsample
	alignx	16,7

.columnloop:
	; -- process the next column block

	movq	mm0, MMWORD [ebx+1*SIZEOF_MMWORD]	; mm0=row[ 0][1]
	movq	mm1, MMWORD [ecx+1*SIZEOF_MMWORD]	; mm1=row[-1][1]
	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]	; mm2=row[+1][1]

	pushpic	ebx
	movpic	ebx, POINTER [gotptr]	; load GOT address

	pxor      mm3,mm3		; mm3=(all 0's)
	movq      mm4,mm0
	punpcklbw mm0,mm3		; mm0=row[ 0][1]( 0 1 2 3)
	punpckhbw mm4,mm3		; mm4=row[ 0][1]( 4 5 6 7)
	movq      mm5,mm1
	punpcklbw mm1,mm3		; mm1=row[-1][1]( 0 1 2 3)
	punpckhbw mm5,mm3		; mm5=row[-1][1]( 4 5 6 7)
	movq      mm6,mm2
	punpcklbw mm2,mm3		; mm2=row[+1][1]( 0 1 2 3)
	punpckhbw mm6,mm3		; mm6=row[+1][1]( 4 5 6 7)

	pmullw	mm0,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]

	paddw	mm1,mm0			; mm1=Int0L=( 0 1 2 3)
	paddw	mm5,mm4			; mm5=Int0H=( 4 5 6 7)
	paddw	mm2,mm0			; mm2=Int1L=( 0 1 2 3)
	paddw	mm6,mm4			; mm6=Int1H=( 4 5 6 7)

	movq	MMWORD [edx+2*SIZEOF_MMWORD], mm1	; temporarily save
	movq	MMWORD [edx+3*SIZEOF_MMWORD], mm5	; the intermediate data
	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm6

	psllq	mm1,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm1=( - - - 0)
	psllq	mm2,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm2=( - - - 0)

	movq	MMWORD [wk(2)], mm1
	movq	MMWORD [wk(3)], mm2

.upsample:
	; -- process the upper row

	movq	mm7, MMWORD [edx+0*SIZEOF_MMWORD]	; mm7=Int0L=( 0 1 2 3)
	movq	mm3, MMWORD [edx+1*SIZEOF_MMWORD]	; mm3=Int0H=( 4 5 6 7)

	movq	mm0,mm7
	movq	mm4,mm3
	psrlq	mm0,2*BYTE_BIT			; mm0=( 1 2 3 -)
	psllq	mm4,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm4=( - - - 4)
	movq	mm5,mm7
	movq	mm6,mm3
	psrlq	mm5,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm5=( 3 - - -)
	psllq	mm6,2*BYTE_BIT			; mm6=( - 4 5 6)

	por	mm0,mm4				; mm0=( 1 2 3 4)
	por	mm5,mm6				; mm5=( 3 4 5 6)

	movq	mm1,mm7
	movq	mm2,mm3
	psllq	mm1,2*BYTE_BIT			; mm1=( - 0 1 2)
	psrlq	mm2,2*BYTE_BIT			; mm2=( 5 6 7 -)
	movq	mm4,mm3
	psrlq	mm4,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm4=( 7 - - -)

	por	mm1, MMWORD [wk(0)]		; mm1=(-1 0 1 2)
	por	mm2, MMWORD [wk(2)]		; mm2=( 5 6 7 8)

	movq	MMWORD [wk(0)], mm4

	pmullw	mm7,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm3,[GOTOFF(ebx,PW_THREE)]
	paddw	mm1,[GOTOFF(ebx,PW_EIGHT)]
	paddw	mm5,[GOTOFF(ebx,PW_EIGHT)]
	paddw	mm0,[GOTOFF(ebx,PW_SEVEN)]
	paddw	mm2,[GOTOFF(ebx,PW_SEVEN)]

	paddw	mm1,mm7
	paddw	mm5,mm3
	psrlw	mm1,4			; mm1=Out0LE=( 0  2  4  6)
	psrlw	mm5,4			; mm5=Out0HE=( 8 10 12 14)
	paddw	mm0,mm7
	paddw	mm2,mm3
	psrlw	mm0,4			; mm0=Out0LO=( 1  3  5  7)
	psrlw	mm2,4			; mm2=Out0HO=( 9 11 13 15)

	psllw	mm0,BYTE_BIT
	psllw	mm2,BYTE_BIT
	por	mm1,mm0			; mm1=Out0L=( 0  1  2  3  4  5  6  7)
	por	mm5,mm2			; mm5=Out0H=( 8  9 10 11 12 13 14 15)

	movq	MMWORD [edx+0*SIZEOF_MMWORD], mm1
	movq	MMWORD [edx+1*SIZEOF_MMWORD], mm5

	; -- process the lower row

	movq	mm6, MMWORD [edi+0*SIZEOF_MMWORD]	; mm6=Int1L=( 0 1 2 3)
	movq	mm4, MMWORD [edi+1*SIZEOF_MMWORD]	; mm4=Int1H=( 4 5 6 7)

	movq	mm7,mm6
	movq	mm3,mm4
	psrlq	mm7,2*BYTE_BIT			; mm7=( 1 2 3 -)
	psllq	mm3,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm3=( - - - 4)
	movq	mm0,mm6
	movq	mm2,mm4
	psrlq	mm0,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm0=( 3 - - -)
	psllq	mm2,2*BYTE_BIT			; mm2=( - 4 5 6)

	por	mm7,mm3				; mm7=( 1 2 3 4)
	por	mm0,mm2				; mm0=( 3 4 5 6)

	movq	mm1,mm6
	movq	mm5,mm4
	psllq	mm1,2*BYTE_BIT			; mm1=( - 0 1 2)
	psrlq	mm5,2*BYTE_BIT			; mm5=( 5 6 7 -)
	movq	mm3,mm4
	psrlq	mm3,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm3=( 7 - - -)

	por	mm1, MMWORD [wk(1)]		; mm1=(-1 0 1 2)
	por	mm5, MMWORD [wk(3)]		; mm5=( 5 6 7 8)

	movq	MMWORD [wk(1)], mm3

	pmullw	mm6,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
	paddw	mm1,[GOTOFF(ebx,PW_EIGHT)]
	paddw	mm0,[GOTOFF(ebx,PW_EIGHT)]
	paddw	mm7,[GOTOFF(ebx,PW_SEVEN)]
	paddw	mm5,[GOTOFF(ebx,PW_SEVEN)]

	paddw	mm1,mm6
	paddw	mm0,mm4
	psrlw	mm1,4			; mm1=Out1LE=( 0  2  4  6)
	psrlw	mm0,4			; mm0=Out1HE=( 8 10 12 14)
	paddw	mm7,mm6
	paddw	mm5,mm4
	psrlw	mm7,4			; mm7=Out1LO=( 1  3  5  7)
	psrlw	mm5,4			; mm5=Out1HO=( 9 11 13 15)

	psllw	mm7,BYTE_BIT
	psllw	mm5,BYTE_BIT
	por	mm1,mm7			; mm1=Out1L=( 0  1  2  3  4  5  6  7)
	por	mm0,mm5			; mm0=Out1H=( 8  9 10 11 12 13 14 15)

	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm1
	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm0

	poppic	ebx

	sub	eax, byte SIZEOF_MMWORD
	add	ecx, byte 1*SIZEOF_MMWORD	; inptr1(above)
	add	ebx, byte 1*SIZEOF_MMWORD	; inptr0
	add	esi, byte 1*SIZEOF_MMWORD	; inptr1(below)
	add	edx, byte 2*SIZEOF_MMWORD	; outptr0
	add	edi, byte 2*SIZEOF_MMWORD	; outptr1
	cmp	eax, byte SIZEOF_MMWORD
	ja	near .columnloop
	test	eax,eax
	jnz	near .columnloop_last

	pop	esi
	pop	edi
	pop	ecx
	pop	eax

	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
	sub	ecx, byte 2			; rowctr
	jg	near .rowloop

	emms		; empty MMX state

.return:
	pop	edi
	pop	esi
;	pop	edx		; need not be preserved
;	pop	ecx		; need not be preserved
	pop	ebx
	mov	esp,ebp		; esp <- aligned ebp
	pop	esp		; esp <- original ebp
	pop	ebp
	ret

%ifdef UPSAMPLE_H1V2_SUPPORTED

; --------------------------------------------------------------------------
;
; Fancy processing for the common case of 1:1 horizontal and 2:1 vertical.
; Again a triangle filter; see comments for h2v1 case, above.
;
; GLOBAL(void)
; jpeg_h1v2_fancy_upsample_mmx (j_decompress_ptr cinfo,
;                               jpeg_component_info * compptr,
;                               JSAMPARRAY input_data,
;                               JSAMPARRAY * output_data_ptr);
;

%define cinfo(b)		(b)+8		; j_decompress_ptr cinfo
%define compptr(b)		(b)+12		; jpeg_component_info * compptr
%define input_data(b)		(b)+16		; JSAMPARRAY input_data
%define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr

%define gotptr		ebp-SIZEOF_POINTER	; void * gotptr

	align	16
	global	EXTN(jpeg_h1v2_fancy_upsample_mmx)

EXTN(jpeg_h1v2_fancy_upsample_mmx):
	push	ebp
	mov	ebp,esp
	pushpic	eax		; make a room for GOT address
	push	ebx
;	push	ecx		; need not be preserved
;	push	edx		; need not be preserved
	push	esi
	push	edi

	get_GOT	ebx			; get GOT address
	movpic	POINTER [gotptr], ebx	; save GOT address

	mov	eax, POINTER [compptr(ebp)]
	mov	eax, JDIMENSION [jcompinfo_downsampled_width(eax)]  ; colctr
	add	eax, byte SIZEOF_MMWORD-1
	and	eax, byte -SIZEOF_MMWORD
	jz	near .return

	mov	ecx, POINTER [cinfo(ebp)]
	mov	ecx, INT [jdstruct_max_v_samp_factor(ecx)]	; rowctr
	test	ecx,ecx
	jz	near .return

	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
	mov	edi, POINTER [output_data_ptr(ebp)]
	mov	edi, JSAMPARRAY [edi]			; output_data
	alignx	16,7
.rowloop:
	push	eax					; colctr
	push	ecx
	push	edi
	push	esi

	mov	ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]	; inptr1(above)
	mov	ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]	; inptr0
	mov	esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]	; inptr1(below)
	mov	edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1

	pxor	mm0,mm0			; mm0=(all 0's)
	alignx	16,7

.columnloop:
	movq	mm1, MMWORD [ebx]	; mm1=row[ 0]( 0 1 2 3 4 5 6 7)
	movq	mm2, MMWORD [ecx]	; mm2=row[-1]( 0 1 2 3 4 5 6 7)
	movq	mm3, MMWORD [esi]	; mm3=row[+1]( 0 1 2 3 4 5 6 7)

	pushpic	ebx
	movpic	ebx, POINTER [gotptr]	; load GOT address

	movq      mm4,mm1
	punpcklbw mm1,mm0		; mm1=row[ 0]( 0 1 2 3)
	punpckhbw mm4,mm0		; mm4=row[ 0]( 4 5 6 7)
	movq      mm5,mm2
	punpcklbw mm2,mm0		; mm2=row[-1]( 0 1 2 3)
	punpckhbw mm5,mm0		; mm5=row[-1]( 4 5 6 7)
	movq      mm6,mm3
	punpcklbw mm3,mm0		; mm3=row[+1]( 0 1 2 3)
	punpckhbw mm6,mm0		; mm6=row[+1]( 4 5 6 7)

	pmullw	mm1,[GOTOFF(ebx,PW_THREE)]
	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
	paddw	mm2,[GOTOFF(ebx,PW_ONE)]
	paddw	mm5,[GOTOFF(ebx,PW_ONE)]
	paddw	mm3,[GOTOFF(ebx,PW_TWO)]
	paddw	mm6,[GOTOFF(ebx,PW_TWO)]

	paddw	mm2,mm1
	paddw	mm5,mm4
	psrlw	mm2,2			; mm2=Out0L=( 0 1 2 3)
	psrlw	mm5,2			; mm5=Out0H=( 4 5 6 7)
	paddw	mm3,mm1
	paddw	mm6,mm4
	psrlw	mm3,2			; mm3=Out1L=( 0 1 2 3)
	psrlw	mm6,2			; mm6=Out1H=( 4 5 6 7)

	packuswb  mm2,mm5		; mm2=Out0=( 0 1 2 3 4 5 6 7)
	packuswb  mm3,mm6		; mm3=Out1=( 0 1 2 3 4 5 6 7)

	movq	MMWORD [edx], mm2
	movq	MMWORD [edi], mm3

	poppic	ebx

	add	ecx, byte 1*SIZEOF_MMWORD	; inptr1(above)
	add	ebx, byte 1*SIZEOF_MMWORD	; inptr0
	add	esi, byte 1*SIZEOF_MMWORD	; inptr1(below)
	add	edx, byte 1*SIZEOF_MMWORD	; outptr0
	add	edi, byte 1*SIZEOF_MMWORD	; outptr1
	sub	eax, byte SIZEOF_MMWORD
	jnz	near .columnloop

	pop	esi
	pop	edi
	pop	ecx
	pop	eax

	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
	sub	ecx, byte 2			; rowctr
	jg	near .rowloop

	emms		; empty MMX state

.return:
	pop	edi
	pop	esi
;	pop	edx		; need not be preserved
;	pop	ecx		; need not be preserved
	pop	ebx
	poppic	eax		; remove gotptr
	pop	ebp
	ret

%endif ; UPSAMPLE_H1V2_SUPPORTED
%endif ; JDSAMPLE_FANCY_MMX_SUPPORTED

%ifdef JDSAMPLE_SIMPLE_MMX_SUPPORTED

%ifndef JDSAMPLE_FANCY_MMX_SUPPORTED
; --------------------------------------------------------------------------
	SECTION	SEG_TEXT
	BITS	32
%endif
;
; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jpeg_h2v1_upsample_mmx (j_decompress_ptr cinfo,
;                         jpeg_component_info * compptr,
;                         JSAMPARRAY input_data,
;                         JSAMPARRAY * output_data_ptr);
;

%define cinfo(b)		(b)+8		; j_decompress_ptr cinfo
%define compptr(b)		(b)+12		; jpeg_component_info * compptr
%define input_data(b)		(b)+16		; JSAMPARRAY input_data
%define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr

	align	16
	global	EXTN(jpeg_h2v1_upsample_mmx)

EXTN(jpeg_h2v1_upsample_mmx):
	push	ebp
	mov	ebp,esp
;	push	ebx		; unused
;	push	ecx		; need not be preserved
;	push	edx		; need not be preserved
	push	esi
	push	edi

	mov	edx, POINTER [cinfo(ebp)]
	mov	edx, JDIMENSION [jdstruct_output_width(edx)]
	add	edx, byte (2*SIZEOF_MMWORD)-1
	and	edx, byte -(2*SIZEOF_MMWORD)
	jz	short .return

	mov	ecx, POINTER [cinfo(ebp)]
	mov	ecx, INT [jdstruct_max_v_samp_factor(ecx)]	; rowctr
	test	ecx,ecx
	jz	short .return

	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
	mov	edi, POINTER [output_data_ptr(ebp)]
	mov	edi, JSAMPARRAY [edi]			; output_data
	alignx	16,7
.rowloop:
	push	edi
	push	esi

	mov	esi, JSAMPROW [esi]		; inptr
	mov	edi, JSAMPROW [edi]		; outptr
	mov	eax,edx				; colctr
	alignx	16,7
.columnloop:

	movq	mm0, MMWORD [esi+0*SIZEOF_MMWORD]

	movq      mm1,mm0
	punpcklbw mm0,mm0
	punpckhbw mm1,mm1

	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm0
	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm1

	sub	eax, byte 2*SIZEOF_MMWORD
	jz	short .nextrow

	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]

	movq      mm3,mm2
	punpcklbw mm2,mm2
	punpckhbw mm3,mm3

	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm3

	sub	eax, byte 2*SIZEOF_MMWORD
	jz	short .nextrow

	add	esi, byte 2*SIZEOF_MMWORD	; inptr
	add	edi, byte 4*SIZEOF_MMWORD	; outptr
	jmp	short .columnloop
	alignx	16,7

.nextrow:
	pop	esi
	pop	edi

	add	esi, byte SIZEOF_JSAMPROW	; input_data
	add	edi, byte SIZEOF_JSAMPROW	; output_data
	dec	ecx				; rowctr
	jg	short .rowloop

	emms		; empty MMX state

.return:
	pop	edi
	pop	esi
;	pop	edx		; need not be preserved
;	pop	ecx		; need not be preserved
;	pop	ebx		; unused
	pop	ebp
	ret

; --------------------------------------------------------------------------
;
; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
; It's still a box filter.
;
; GLOBAL(void)
; jpeg_h2v2_upsample_mmx (j_decompress_ptr cinfo,
;                         jpeg_component_info * compptr,
;                         JSAMPARRAY input_data,
;                         JSAMPARRAY * output_data_ptr);
;

%define cinfo(b)		(b)+8		; j_decompress_ptr cinfo
%define compptr(b)		(b)+12		; jpeg_component_info * compptr
%define input_data(b)		(b)+16		; JSAMPARRAY input_data
%define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr

	align	16
	global	EXTN(jpeg_h2v2_upsample_mmx)

EXTN(jpeg_h2v2_upsample_mmx):
	push	ebp
	mov	ebp,esp
	push	ebx
;	push	ecx		; need not be preserved
;	push	edx		; need not be preserved
	push	esi
	push	edi

	mov	edx, POINTER [cinfo(ebp)]
	mov	edx, JDIMENSION [jdstruct_output_width(edx)]
	add	edx, byte (2*SIZEOF_MMWORD)-1
	and	edx, byte -(2*SIZEOF_MMWORD)
	jz	near .return

	mov	ecx, POINTER [cinfo(ebp)]
	mov	ecx, INT [jdstruct_max_v_samp_factor(ecx)]	; rowctr
	test	ecx,ecx
	jz	short .return

	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
	mov	edi, POINTER [output_data_ptr(ebp)]
	mov	edi, JSAMPARRAY [edi]			; output_data
	alignx	16,7
.rowloop:
	push	edi
	push	esi

	mov	esi, JSAMPROW [esi]			; inptr
	mov	ebx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1
	mov	eax,edx					; colctr
	alignx	16,7
.columnloop:

	movq	mm0, MMWORD [esi+0*SIZEOF_MMWORD]

	movq      mm1,mm0
	punpcklbw mm0,mm0
	punpckhbw mm1,mm1

	movq	MMWORD [ebx+0*SIZEOF_MMWORD], mm0
	movq	MMWORD [ebx+1*SIZEOF_MMWORD], mm1
	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm0
	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm1

	sub	eax, byte 2*SIZEOF_MMWORD
	jz	short .nextrow

	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]

	movq      mm3,mm2
	punpcklbw mm2,mm2
	punpckhbw mm3,mm3

	movq	MMWORD [ebx+2*SIZEOF_MMWORD], mm2
	movq	MMWORD [ebx+3*SIZEOF_MMWORD], mm3
	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm3

	sub	eax, byte 2*SIZEOF_MMWORD
	jz	short .nextrow

	add	esi, byte 2*SIZEOF_MMWORD	; inptr
	add	ebx, byte 4*SIZEOF_MMWORD	; outptr0
	add	edi, byte 4*SIZEOF_MMWORD	; outptr1
	jmp	short .columnloop
	alignx	16,7

.nextrow:
	pop	esi
	pop	edi

	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
	sub	ecx, byte 2			; rowctr
	jg	short .rowloop

	emms		; empty MMX state

.return:
	pop	edi
	pop	esi
;	pop	edx		; need not be preserved
;	pop	ecx		; need not be preserved
	pop	ebx
	pop	ebp
	ret

%endif ; JDSAMPLE_SIMPLE_MMX_SUPPORTED