| // Copyright ©2017 The Gonum Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // +build !noasm,!gccgo,!safe |
| |
| #include "textflag.h" |
| |
| #define SIZE 8 |
| |
| #define M_DIM n+8(FP) |
| #define M CX |
| #define N_DIM m+0(FP) |
| #define N BX |
| |
| #define TMP1 R14 |
| #define TMP2 R15 |
| |
| #define X_PTR SI |
| #define X x_base+56(FP) |
| #define Y_PTR DX |
| #define Y y_base+96(FP) |
| #define A_ROW AX |
| #define A_PTR DI |
| |
| #define INC_X R8 |
| #define INC3_X R9 |
| |
| #define INC_Y R10 |
| #define INC3_Y R11 |
| |
| #define LDA R12 |
| #define LDA3 R13 |
| |
| #define ALPHA X15 |
| #define BETA X14 |
| |
| #define INIT4 \ |
| MOVDDUP (X_PTR), X8 \ |
| MOVDDUP (X_PTR)(INC_X*1), X9 \ |
| MOVDDUP (X_PTR)(INC_X*2), X10 \ |
| MOVDDUP (X_PTR)(INC3_X*1), X11 \ |
| MULPD ALPHA, X8 \ |
| MULPD ALPHA, X9 \ |
| MULPD ALPHA, X10 \ |
| MULPD ALPHA, X11 |
| |
| #define INIT2 \ |
| MOVDDUP (X_PTR), X8 \ |
| MOVDDUP (X_PTR)(INC_X*1), X9 \ |
| MULPD ALPHA, X8 \ |
| MULPD ALPHA, X9 |
| |
| #define INIT1 \ |
| MOVDDUP (X_PTR), X8 \ |
| MULPD ALPHA, X8 |
| |
| #define KERNEL_LOAD4 \ |
| MOVUPS (Y_PTR), X0 \ |
| MOVUPS 2*SIZE(Y_PTR), X1 |
| |
| #define KERNEL_LOAD2 \ |
| MOVUPS (Y_PTR), X0 |
| |
| #define KERNEL_LOAD4_INC \ |
| MOVSD (Y_PTR), X0 \ |
| MOVHPD (Y_PTR)(INC_Y*1), X0 \ |
| MOVSD (Y_PTR)(INC_Y*2), X1 \ |
| MOVHPD (Y_PTR)(INC3_Y*1), X1 |
| |
| #define KERNEL_LOAD2_INC \ |
| MOVSD (Y_PTR), X0 \ |
| MOVHPD (Y_PTR)(INC_Y*1), X0 |
| |
| #define KERNEL_4x4 \ |
| MOVUPS (A_PTR), X4 \ |
| MOVUPS 2*SIZE(A_PTR), X5 \ |
| MOVUPS (A_PTR)(LDA*1), X6 \ |
| MOVUPS 2*SIZE(A_PTR)(LDA*1), X7 \ |
| MULPD X8, X4 \ |
| MULPD X8, X5 \ |
| MULPD X9, X6 \ |
| MULPD X9, X7 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X1 \ |
| ADDPD X6, X0 \ |
| ADDPD X7, X1 \ |
| MOVUPS (A_PTR)(LDA*2), X4 \ |
| MOVUPS 2*SIZE(A_PTR)(LDA*2), X5 \ |
| MOVUPS (A_PTR)(LDA3*1), X6 \ |
| MOVUPS 2*SIZE(A_PTR)(LDA3*1), X7 \ |
| MULPD X10, X4 \ |
| MULPD X10, X5 \ |
| MULPD X11, X6 \ |
| MULPD X11, X7 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X1 \ |
| ADDPD X6, X0 \ |
| ADDPD X7, X1 \ |
| ADDQ $4*SIZE, A_PTR |
| |
| #define KERNEL_4x2 \ |
| MOVUPS (A_PTR), X4 \ |
| MOVUPS 2*SIZE(A_PTR), X5 \ |
| MOVUPS (A_PTR)(LDA*1), X6 \ |
| MOVUPS 2*SIZE(A_PTR)(LDA*1), X7 \ |
| MULPD X8, X4 \ |
| MULPD X8, X5 \ |
| MULPD X9, X6 \ |
| MULPD X9, X7 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X1 \ |
| ADDPD X6, X0 \ |
| ADDPD X7, X1 \ |
| ADDQ $4*SIZE, A_PTR |
| |
| #define KERNEL_4x1 \ |
| MOVUPS (A_PTR), X4 \ |
| MOVUPS 2*SIZE(A_PTR), X5 \ |
| MULPD X8, X4 \ |
| MULPD X8, X5 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X1 \ |
| ADDQ $4*SIZE, A_PTR |
| |
| #define STORE4 \ |
| MOVUPS X0, (Y_PTR) \ |
| MOVUPS X1, 2*SIZE(Y_PTR) |
| |
| #define STORE4_INC \ |
| MOVLPD X0, (Y_PTR) \ |
| MOVHPD X0, (Y_PTR)(INC_Y*1) \ |
| MOVLPD X1, (Y_PTR)(INC_Y*2) \ |
| MOVHPD X1, (Y_PTR)(INC3_Y*1) |
| |
| #define KERNEL_2x4 \ |
| MOVUPS (A_PTR), X4 \ |
| MOVUPS (A_PTR)(LDA*1), X5 \ |
| MOVUPS (A_PTR)(LDA*2), X6 \ |
| MOVUPS (A_PTR)(LDA3*1), X7 \ |
| MULPD X8, X4 \ |
| MULPD X9, X5 \ |
| MULPD X10, X6 \ |
| MULPD X11, X7 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X0 \ |
| ADDPD X6, X0 \ |
| ADDPD X7, X0 \ |
| ADDQ $2*SIZE, A_PTR |
| |
| #define KERNEL_2x2 \ |
| MOVUPS (A_PTR), X4 \ |
| MOVUPS (A_PTR)(LDA*1), X5 \ |
| MULPD X8, X4 \ |
| MULPD X9, X5 \ |
| ADDPD X4, X0 \ |
| ADDPD X5, X0 \ |
| ADDQ $2*SIZE, A_PTR |
| |
| #define KERNEL_2x1 \ |
| MOVUPS (A_PTR), X4 \ |
| MULPD X8, X4 \ |
| ADDPD X4, X0 \ |
| ADDQ $2*SIZE, A_PTR |
| |
| #define STORE2 \ |
| MOVUPS X0, (Y_PTR) |
| |
| #define STORE2_INC \ |
| MOVLPD X0, (Y_PTR) \ |
| MOVHPD X0, (Y_PTR)(INC_Y*1) |
| |
| #define KERNEL_1x4 \ |
| MOVSD (Y_PTR), X0 \ |
| MOVSD (A_PTR), X4 \ |
| MOVSD (A_PTR)(LDA*1), X5 \ |
| MOVSD (A_PTR)(LDA*2), X6 \ |
| MOVSD (A_PTR)(LDA3*1), X7 \ |
| MULSD X8, X4 \ |
| MULSD X9, X5 \ |
| MULSD X10, X6 \ |
| MULSD X11, X7 \ |
| ADDSD X4, X0 \ |
| ADDSD X5, X0 \ |
| ADDSD X6, X0 \ |
| ADDSD X7, X0 \ |
| MOVSD X0, (Y_PTR) \ |
| ADDQ $SIZE, A_PTR |
| |
| #define KERNEL_1x2 \ |
| MOVSD (Y_PTR), X0 \ |
| MOVSD (A_PTR), X4 \ |
| MOVSD (A_PTR)(LDA*1), X5 \ |
| MULSD X8, X4 \ |
| MULSD X9, X5 \ |
| ADDSD X4, X0 \ |
| ADDSD X5, X0 \ |
| MOVSD X0, (Y_PTR) \ |
| ADDQ $SIZE, A_PTR |
| |
| #define KERNEL_1x1 \ |
| MOVSD (Y_PTR), X0 \ |
| MOVSD (A_PTR), X4 \ |
| MULSD X8, X4 \ |
| ADDSD X4, X0 \ |
| MOVSD X0, (Y_PTR) \ |
| ADDQ $SIZE, A_PTR |
| |
| #define SCALE_8(PTR, SCAL) \ |
| MOVUPS (PTR), X0 \ |
| MOVUPS 16(PTR), X1 \ |
| MOVUPS 32(PTR), X2 \ |
| MOVUPS 48(PTR), X3 \ |
| MULPD SCAL, X0 \ |
| MULPD SCAL, X1 \ |
| MULPD SCAL, X2 \ |
| MULPD SCAL, X3 \ |
| MOVUPS X0, (PTR) \ |
| MOVUPS X1, 16(PTR) \ |
| MOVUPS X2, 32(PTR) \ |
| MOVUPS X3, 48(PTR) |
| |
| #define SCALE_4(PTR, SCAL) \ |
| MOVUPS (PTR), X0 \ |
| MOVUPS 16(PTR), X1 \ |
| MULPD SCAL, X0 \ |
| MULPD SCAL, X1 \ |
| MOVUPS X0, (PTR) \ |
| MOVUPS X1, 16(PTR) \ |
| |
| #define SCALE_2(PTR, SCAL) \ |
| MOVUPS (PTR), X0 \ |
| MULPD SCAL, X0 \ |
| MOVUPS X0, (PTR) \ |
| |
| #define SCALE_1(PTR, SCAL) \ |
| MOVSD (PTR), X0 \ |
| MULSD SCAL, X0 \ |
| MOVSD X0, (PTR) \ |
| |
| #define SCALEINC_4(PTR, INC, INC3, SCAL) \ |
| MOVSD (PTR), X0 \ |
| MOVSD (PTR)(INC*1), X1 \ |
| MOVSD (PTR)(INC*2), X2 \ |
| MOVSD (PTR)(INC3*1), X3 \ |
| MULSD SCAL, X0 \ |
| MULSD SCAL, X1 \ |
| MULSD SCAL, X2 \ |
| MULSD SCAL, X3 \ |
| MOVSD X0, (PTR) \ |
| MOVSD X1, (PTR)(INC*1) \ |
| MOVSD X2, (PTR)(INC*2) \ |
| MOVSD X3, (PTR)(INC3*1) |
| |
| #define SCALEINC_2(PTR, INC, SCAL) \ |
| MOVSD (PTR), X0 \ |
| MOVSD (PTR)(INC*1), X1 \ |
| MULSD SCAL, X0 \ |
| MULSD SCAL, X1 \ |
| MOVSD X0, (PTR) \ |
| MOVSD X1, (PTR)(INC*1) |
| |
| // func GemvT(m, n int, |
| // alpha float64, |
| // a []float64, lda int, |
| // x []float64, incX int, |
| // beta float64, |
| // y []float64, incY int) |
| TEXT ·GemvT(SB), NOSPLIT, $32-128 |
| MOVQ M_DIM, M |
| MOVQ N_DIM, N |
| CMPQ M, $0 |
| JE end |
| CMPQ N, $0 |
| JE end |
| |
| MOVDDUP alpha+16(FP), ALPHA |
| |
| MOVQ x_base+56(FP), X_PTR |
| MOVQ y_base+96(FP), Y_PTR |
| MOVQ a_base+24(FP), A_ROW |
| MOVQ incY+120(FP), INC_Y // INC_Y = incY * sizeof(float64) |
| MOVQ lda+48(FP), LDA // LDA = LDA * sizeof(float64) |
| SHLQ $3, LDA |
| LEAQ (LDA)(LDA*2), LDA3 // LDA3 = LDA * 3 |
| MOVQ A_ROW, A_PTR |
| |
| MOVQ incX+80(FP), INC_X // INC_X = incX * sizeof(float64) |
| |
| XORQ TMP2, TMP2 |
| MOVQ N, TMP1 |
| SUBQ $1, TMP1 |
| NEGQ TMP1 |
| IMULQ INC_X, TMP1 |
| CMPQ INC_X, $0 |
| CMOVQLT TMP1, TMP2 |
| LEAQ (X_PTR)(TMP2*SIZE), X_PTR |
| MOVQ X_PTR, X |
| |
| SHLQ $3, INC_X |
| LEAQ (INC_X)(INC_X*2), INC3_X // INC3_X = INC_X * 3 |
| |
| CMPQ incY+120(FP), $1 // Check for dense vector Y (fast-path) |
| JNE inc |
| |
| MOVSD $1.0, X0 |
| COMISD beta+88(FP), X0 |
| JE gemv_start |
| |
| MOVSD $0.0, X0 |
| COMISD beta+88(FP), X0 |
| JE gemv_clear |
| |
| MOVDDUP beta+88(FP), BETA |
| SHRQ $3, M |
| JZ scal4 |
| |
| scal8: |
| SCALE_8(Y_PTR, BETA) |
| ADDQ $8*SIZE, Y_PTR |
| DECQ M |
| JNZ scal8 |
| |
| scal4: |
| TESTQ $4, M_DIM |
| JZ scal2 |
| SCALE_4(Y_PTR, BETA) |
| ADDQ $4*SIZE, Y_PTR |
| |
| scal2: |
| TESTQ $2, M_DIM |
| JZ scal1 |
| SCALE_2(Y_PTR, BETA) |
| ADDQ $2*SIZE, Y_PTR |
| |
| scal1: |
| TESTQ $1, M_DIM |
| JZ prep_end |
| SCALE_1(Y_PTR, BETA) |
| |
| JMP prep_end |
| |
| gemv_clear: // beta == 0 is special cased to clear memory (no nan handling) |
| XORPS X0, X0 |
| XORPS X1, X1 |
| XORPS X2, X2 |
| XORPS X3, X3 |
| |
| SHRQ $3, M |
| JZ clear4 |
| |
| clear8: |
| MOVUPS X0, (Y_PTR) |
| MOVUPS X1, 16(Y_PTR) |
| MOVUPS X2, 32(Y_PTR) |
| MOVUPS X3, 48(Y_PTR) |
| ADDQ $8*SIZE, Y_PTR |
| DECQ M |
| JNZ clear8 |
| |
| clear4: |
| TESTQ $4, M_DIM |
| JZ clear2 |
| MOVUPS X0, (Y_PTR) |
| MOVUPS X1, 16(Y_PTR) |
| ADDQ $4*SIZE, Y_PTR |
| |
| clear2: |
| TESTQ $2, M_DIM |
| JZ clear1 |
| MOVUPS X0, (Y_PTR) |
| ADDQ $2*SIZE, Y_PTR |
| |
| clear1: |
| TESTQ $1, M_DIM |
| JZ prep_end |
| MOVSD X0, (Y_PTR) |
| |
| prep_end: |
| MOVQ Y, Y_PTR |
| MOVQ M_DIM, M |
| |
| gemv_start: |
| SHRQ $2, N |
| JZ c2 |
| |
| c4: |
| // LOAD 4 |
| INIT4 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ c4r2 |
| |
| c4r4: |
| // 4x4 KERNEL |
| KERNEL_LOAD4 |
| KERNEL_4x4 |
| STORE4 |
| |
| ADDQ $4*SIZE, Y_PTR |
| |
| DECQ M |
| JNZ c4r4 |
| |
| c4r2: |
| TESTQ $2, M_DIM |
| JZ c4r1 |
| |
| // 4x2 KERNEL |
| KERNEL_LOAD2 |
| KERNEL_2x4 |
| STORE2 |
| |
| ADDQ $2*SIZE, Y_PTR |
| |
| c4r1: |
| TESTQ $1, M_DIM |
| JZ c4end |
| |
| // 4x1 KERNEL |
| KERNEL_1x4 |
| |
| ADDQ $SIZE, Y_PTR |
| |
| c4end: |
| LEAQ (X_PTR)(INC_X*4), X_PTR |
| MOVQ Y, Y_PTR |
| LEAQ (A_ROW)(LDA*4), A_ROW |
| MOVQ A_ROW, A_PTR |
| |
| DECQ N |
| JNZ c4 |
| |
| c2: |
| TESTQ $2, N_DIM |
| JZ c1 |
| |
| // LOAD 2 |
| INIT2 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ c2r2 |
| |
| c2r4: |
| // 2x4 KERNEL |
| KERNEL_LOAD4 |
| KERNEL_4x2 |
| STORE4 |
| |
| ADDQ $4*SIZE, Y_PTR |
| |
| DECQ M |
| JNZ c2r4 |
| |
| c2r2: |
| TESTQ $2, M_DIM |
| JZ c2r1 |
| |
| // 2x2 KERNEL |
| KERNEL_LOAD2 |
| KERNEL_2x2 |
| STORE2 |
| |
| ADDQ $2*SIZE, Y_PTR |
| |
| c2r1: |
| TESTQ $1, M_DIM |
| JZ c2end |
| |
| // 2x1 KERNEL |
| KERNEL_1x2 |
| |
| ADDQ $SIZE, Y_PTR |
| |
| c2end: |
| LEAQ (X_PTR)(INC_X*2), X_PTR |
| MOVQ Y, Y_PTR |
| LEAQ (A_ROW)(LDA*2), A_ROW |
| MOVQ A_ROW, A_PTR |
| |
| c1: |
| TESTQ $1, N_DIM |
| JZ end |
| |
| // LOAD 1 |
| INIT1 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ c1r2 |
| |
| c1r4: |
| // 1x4 KERNEL |
| KERNEL_LOAD4 |
| KERNEL_4x1 |
| STORE4 |
| |
| ADDQ $4*SIZE, Y_PTR |
| |
| DECQ M |
| JNZ c1r4 |
| |
| c1r2: |
| TESTQ $2, M_DIM |
| JZ c1r1 |
| |
| // 1x2 KERNEL |
| KERNEL_LOAD2 |
| KERNEL_2x1 |
| STORE2 |
| |
| ADDQ $2*SIZE, Y_PTR |
| |
| c1r1: |
| TESTQ $1, M_DIM |
| JZ end |
| |
| // 1x1 KERNEL |
| KERNEL_1x1 |
| |
| end: |
| RET |
| |
| inc: // Algorithm for incX != 0 ( split loads in kernel ) |
| XORQ TMP2, TMP2 |
| MOVQ M, TMP1 |
| SUBQ $1, TMP1 |
| IMULQ INC_Y, TMP1 |
| NEGQ TMP1 |
| CMPQ INC_Y, $0 |
| CMOVQLT TMP1, TMP2 |
| LEAQ (Y_PTR)(TMP2*SIZE), Y_PTR |
| MOVQ Y_PTR, Y |
| |
| SHLQ $3, INC_Y |
| LEAQ (INC_Y)(INC_Y*2), INC3_Y // INC3_Y = INC_Y * 3 |
| |
| MOVSD $1.0, X0 |
| COMISD beta+88(FP), X0 |
| JE inc_gemv_start |
| |
| MOVSD $0.0, X0 |
| COMISD beta+88(FP), X0 |
| JE inc_gemv_clear |
| |
| MOVDDUP beta+88(FP), BETA |
| SHRQ $2, M |
| JZ inc_scal2 |
| |
| inc_scal4: |
| SCALEINC_4(Y_PTR, INC_Y, INC3_Y, BETA) |
| LEAQ (Y_PTR)(INC_Y*4), Y_PTR |
| DECQ M |
| JNZ inc_scal4 |
| |
| inc_scal2: |
| TESTQ $2, M_DIM |
| JZ inc_scal1 |
| |
| SCALEINC_2(Y_PTR, INC_Y, BETA) |
| LEAQ (Y_PTR)(INC_Y*2), Y_PTR |
| |
| inc_scal1: |
| TESTQ $1, M_DIM |
| JZ inc_prep_end |
| SCALE_1(Y_PTR, BETA) |
| |
| JMP inc_prep_end |
| |
| inc_gemv_clear: // beta == 0 is special-cased to clear memory (no nan handling) |
| XORPS X0, X0 |
| XORPS X1, X1 |
| XORPS X2, X2 |
| XORPS X3, X3 |
| |
| SHRQ $2, M |
| JZ inc_clear2 |
| |
| inc_clear4: |
| MOVSD X0, (Y_PTR) |
| MOVSD X1, (Y_PTR)(INC_Y*1) |
| MOVSD X2, (Y_PTR)(INC_Y*2) |
| MOVSD X3, (Y_PTR)(INC3_Y*1) |
| LEAQ (Y_PTR)(INC_Y*4), Y_PTR |
| DECQ M |
| JNZ inc_clear4 |
| |
| inc_clear2: |
| TESTQ $2, M_DIM |
| JZ inc_clear1 |
| MOVSD X0, (Y_PTR) |
| MOVSD X1, (Y_PTR)(INC_Y*1) |
| LEAQ (Y_PTR)(INC_Y*2), Y_PTR |
| |
| inc_clear1: |
| TESTQ $1, M_DIM |
| JZ inc_prep_end |
| MOVSD X0, (Y_PTR) |
| |
| inc_prep_end: |
| MOVQ Y, Y_PTR |
| MOVQ M_DIM, M |
| |
| inc_gemv_start: |
| SHRQ $2, N |
| JZ inc_c2 |
| |
| inc_c4: |
| // LOAD 4 |
| INIT4 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ inc_c4r2 |
| |
| inc_c4r4: |
| // 4x4 KERNEL |
| KERNEL_LOAD4_INC |
| KERNEL_4x4 |
| STORE4_INC |
| |
| LEAQ (Y_PTR)(INC_Y*4), Y_PTR |
| |
| DECQ M |
| JNZ inc_c4r4 |
| |
| inc_c4r2: |
| TESTQ $2, M_DIM |
| JZ inc_c4r1 |
| |
| // 4x2 KERNEL |
| KERNEL_LOAD2_INC |
| KERNEL_2x4 |
| STORE2_INC |
| |
| LEAQ (Y_PTR)(INC_Y*2), Y_PTR |
| |
| inc_c4r1: |
| TESTQ $1, M_DIM |
| JZ inc_c4end |
| |
| // 4x1 KERNEL |
| KERNEL_1x4 |
| |
| ADDQ INC_Y, Y_PTR |
| |
| inc_c4end: |
| LEAQ (X_PTR)(INC_X*4), X_PTR |
| MOVQ Y, Y_PTR |
| LEAQ (A_ROW)(LDA*4), A_ROW |
| MOVQ A_ROW, A_PTR |
| |
| DECQ N |
| JNZ inc_c4 |
| |
| inc_c2: |
| TESTQ $2, N_DIM |
| JZ inc_c1 |
| |
| // LOAD 2 |
| INIT2 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ inc_c2r2 |
| |
| inc_c2r4: |
| // 2x4 KERNEL |
| KERNEL_LOAD4_INC |
| KERNEL_4x2 |
| STORE4_INC |
| |
| LEAQ (Y_PTR)(INC_Y*4), Y_PTR |
| DECQ M |
| JNZ inc_c2r4 |
| |
| inc_c2r2: |
| TESTQ $2, M_DIM |
| JZ inc_c2r1 |
| |
| // 2x2 KERNEL |
| KERNEL_LOAD2_INC |
| KERNEL_2x2 |
| STORE2_INC |
| |
| LEAQ (Y_PTR)(INC_Y*2), Y_PTR |
| |
| inc_c2r1: |
| TESTQ $1, M_DIM |
| JZ inc_c2end |
| |
| // 2x1 KERNEL |
| KERNEL_1x2 |
| |
| ADDQ INC_Y, Y_PTR |
| |
| inc_c2end: |
| LEAQ (X_PTR)(INC_X*2), X_PTR |
| MOVQ Y, Y_PTR |
| LEAQ (A_ROW)(LDA*2), A_ROW |
| MOVQ A_ROW, A_PTR |
| |
| inc_c1: |
| TESTQ $1, N_DIM |
| JZ inc_end |
| |
| // LOAD 1 |
| INIT1 |
| |
| MOVQ M_DIM, M |
| SHRQ $2, M |
| JZ inc_c1r2 |
| |
| inc_c1r4: |
| // 1x4 KERNEL |
| KERNEL_LOAD4_INC |
| KERNEL_4x1 |
| STORE4_INC |
| |
| LEAQ (Y_PTR)(INC_Y*4), Y_PTR |
| DECQ M |
| JNZ inc_c1r4 |
| |
| inc_c1r2: |
| TESTQ $2, M_DIM |
| JZ inc_c1r1 |
| |
| // 1x2 KERNEL |
| KERNEL_LOAD2_INC |
| KERNEL_2x1 |
| STORE2_INC |
| |
| LEAQ (Y_PTR)(INC_Y*2), Y_PTR |
| |
| inc_c1r1: |
| TESTQ $1, M_DIM |
| JZ inc_end |
| |
| // 1x1 KERNEL |
| KERNEL_1x1 |
| |
| inc_end: |
| RET |
