| |
| /* |
| * Test 128 and 256-bit load and store intrinsics, |
| * with masked and zero-masked forms, by comparing |
| * their output with the corresponding 512-bit intrinsic. |
| * |
| * This test was created to check the correctness |
| * of the following intrinsics support: |
| * _mm512_load_si512() |
| * _mm512_store_si512() |
| * _mm256_storeu_ps() |
| * _mm256_storeu_si256() |
| * _mm_storeu_ps() |
| * _mm_storeu_si128() |
| */ |
| |
| #include "m512_test_util.h" |
| #include <stdio.h> |
| |
| V512 fsrc1; |
| V512 fsrc2; |
| V512 fsrc_non_negative; |
| V512 fsrc_arr[2]; |
| |
| V512 dsrc1; |
| V512 dsrc2; |
| V512 dsrc_non_negative; |
| V512 dsrc_arr[2]; |
| |
| V512 isrc1; |
| V512 isrc2; |
| V512 isrc_arr[2]; |
| |
| static void NOINLINE init() { |
| volatile int i; |
| |
| for (i = 0; i < 16; i++) { |
| fsrc1.f32[i] = (float)(i + 1); |
| fsrc2.f32[i] = 4.0f * (float)(i + 1); |
| fsrc_non_negative.f32[i] = 2.0f * (float)(i); |
| isrc2.s32[i] = (3 * i) - 17; |
| } |
| |
| fsrc_arr[0] = fsrc2; |
| fsrc_arr[1] = fsrc1; |
| |
| for (i = 0; i < 64; i++) { |
| isrc1.s8[i] = 2 * i + 1; |
| if (i % 3) { |
| isrc1.s8[i] = -isrc1.s8[i]; |
| } |
| } |
| |
| isrc_arr[0] = isrc2; |
| isrc_arr[1] = isrc1; |
| |
| for (i = 0; i < 8; i++) { |
| dsrc1.f64[i] = (double)(-i - 1); |
| dsrc2.f64[i] = 3.0 * (double)(-i - 1); |
| dsrc_non_negative.f64[i] = 4.0f * (double)(i); |
| } |
| |
| dsrc_arr[0] = dsrc2; |
| dsrc_arr[1] = dsrc1; |
| } |
| |
| /* |
| * Use "soft update" between tests to make compiler think src was updated. |
| * Prevents PRE'ing a load of src, thus allowing ciscization. |
| * Also prevents PRE'ing intrinsic operations, ensuring we |
| * execute the intended instructions. |
| */ |
| volatile int vol0 = 0; |
| #define soft_v512_update(var) (var).xmmi[vol0] = (var).xmmi[vol0] |
| |
| void NOINLINE do_load_and_loadu_pd() { |
| V512 xmm_res, ymm_res, zmm_res; |
| __mmask8 k8 = 0x7e; |
| |
| /* Non-masked. */ |
| |
| soft_v512_update(dsrc1); |
| zmm_res.zmmd = _mm512_load_pd(&dsrc1.zmmd); |
| |
| soft_v512_update(dsrc_arr[0]); |
| zmm_res.zmmd = _mm512_loadu_pd(&dsrc_arr[0].f64[1]); |
| |
| /* Masked. */ |
| |
| zmm_res.zmmd = _mm512_setzero_pd(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(dsrc1); |
| zmm_res.zmmd = _mm512_mask_load_pd(zmm_res.zmmd, k8, &dsrc1.zmmd); |
| |
| zmm_res.zmmd = _mm512_setzero_pd(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(dsrc_arr[0]); |
| zmm_res.zmmd = _mm512_mask_loadu_pd(zmm_res.zmmd, k8, &dsrc_arr[0].f64[3]); |
| |
| /* Zero-masked. */ |
| |
| zmm_res.zmmd = _mm512_set1_pd(1.0); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(dsrc1); |
| zmm_res.zmmd = _mm512_maskz_load_pd(k8, &dsrc1.zmmd); |
| soft_v512_update(dsrc1); |
| |
| zmm_res.zmmd = _mm512_set1_pd(1.0); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| } |
| |
| void NOINLINE do_store_and_storeu_pd() { |
| V512 xmm_res[2], ymm_res[2], zmm_res[2]; |
| __mmask8 k8 = 0xef; |
| |
| /* Non-masked. */ |
| |
| zmm_res[0].zmmd = _mm512_set1_pd(1.0); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(dsrc1); |
| _mm512_store_pd(&zmm_res[0].zmmd, dsrc1.zmmd); |
| |
| soft_v512_update(dsrc1); |
| _mm512_storeu_pd(&zmm_res[0].f64[1], dsrc1.zmmd); |
| |
| /* Masked. */ |
| |
| zmm_res[0].zmmd = _mm512_set1_pd(1.0); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(dsrc1); |
| _mm512_mask_store_pd(&zmm_res[0].zmmd, k8, dsrc1.zmmd); |
| |
| zmm_res[0].zmmd = _mm512_set1_pd(1.0); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(dsrc1); |
| _mm512_mask_storeu_pd(&zmm_res[0].f64[1], k8, dsrc1.zmmd); |
| } |
| |
| void NOINLINE do_load_and_loadu_ps() { |
| V512 xmm_res, ymm_res, zmm_res; |
| __mmask16 k16 = 0xff7e; |
| __mmask8 k8 = (__mmask8)k16; |
| |
| /* Non-masked. */ |
| |
| soft_v512_update(fsrc1); |
| zmm_res.zmm = _mm512_load_ps(&fsrc1.zmm); |
| |
| soft_v512_update(fsrc_arr[0]); |
| zmm_res.zmm = _mm512_loadu_ps(&fsrc_arr[0].f32[3]); |
| |
| /* Masked. */ |
| |
| zmm_res.zmm = _mm512_setzero_ps(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(fsrc1); |
| zmm_res.zmm = _mm512_mask_load_ps(zmm_res.zmm, k16, &fsrc1.zmm); |
| |
| zmm_res.zmm = _mm512_setzero_ps(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(fsrc_arr[0]); |
| zmm_res.zmm = _mm512_mask_loadu_ps(zmm_res.zmm, k16, &fsrc_arr[0].f32[5]); |
| |
| /* Zero-masked. */ |
| |
| zmm_res.zmm = _mm512_set1_ps(1.0f); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(fsrc1); |
| zmm_res.zmm = _mm512_maskz_load_ps(k16, &fsrc1.zmm); |
| |
| zmm_res.zmm = _mm512_set1_ps(1.0f); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(fsrc_arr[0]); |
| zmm_res.zmm = _mm512_maskz_loadu_ps(k16, &fsrc_arr[0].f32[5]); |
| } |
| |
| void NOINLINE do_store_and_storeu_ps() { |
| V512 xmm_res[2], ymm_res[2], zmm_res[2]; |
| __mmask16 k16 = 0xffef; |
| __mmask8 k8 = (__mmask8)k16; |
| |
| /* Non-masked. */ |
| |
| zmm_res[0].zmm = _mm512_set1_ps(1.0f); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(fsrc1); |
| _mm512_store_ps(&zmm_res[0].zmm, fsrc1.zmm); |
| |
| soft_v512_update(fsrc1); |
| _mm512_storeu_ps(&zmm_res[0].f32[1], fsrc1.zmm); |
| soft_v512_update(fsrc1); |
| _mm256_storeu_ps(&ymm_res[0].f32[1], fsrc1.ymm[0]); |
| soft_v512_update(fsrc1); |
| _mm_storeu_ps(&xmm_res[0].f32[1], fsrc1.xmm[0]); |
| |
| check_equal_nsf(&ymm_res[0].f32[1], &zmm_res[0].f32[1], 8, "_mm256_storeu_ps", |
| __LINE__); |
| check_equal_nsf(&xmm_res[0].f32[1], &zmm_res[0].f32[1], 4, "_mm_storeu_ps", |
| __LINE__); |
| |
| /* Masked. */ |
| |
| zmm_res[0].zmm = _mm512_set1_ps(1.0f); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(fsrc1); |
| _mm512_mask_store_ps(&zmm_res[0].zmm, k8, fsrc1.zmm); |
| |
| zmm_res[0].zmm = _mm512_set1_ps(1.0f); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(fsrc1); |
| _mm512_mask_storeu_ps(&zmm_res[0].f32[1], k8, fsrc1.zmm); |
| } |
| |
| void NOINLINE do_load_and_loadu_epi32() { |
| V512 xmm_res, ymm_res, zmm_res; |
| __mmask16 k16 = 0xffef; |
| __mmask8 k8 = (__mmask8)k16; |
| |
| /* Non-masked. */ |
| |
| soft_v512_update(isrc1); |
| zmm_res.zmmi = _mm512_load_epi32(&isrc1.zmmi); |
| |
| soft_v512_update(isrc1); |
| ymm_res.zmmi = _mm512_load_si512(&isrc1.zmmi); |
| check_equal_nd(&ymm_res, &zmm_res, 16, "_mm512_load_si512", __LINE__); |
| |
| soft_v512_update(isrc_arr[0]); |
| zmm_res.zmmi = _mm512_loadu_si512(&isrc_arr[0].s32[1]); |
| |
| /* Masked. */ |
| |
| zmm_res.zmmi = _mm512_setzero_epi32(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc1); |
| zmm_res.zmmi = _mm512_mask_load_epi32(zmm_res.zmmi, k16, &isrc1.zmmi); |
| |
| zmm_res.zmmi = _mm512_setzero_epi32(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc_arr[0]); |
| zmm_res.zmmi = |
| _mm512_mask_loadu_epi32(zmm_res.zmmi, k16, &isrc_arr[0].s32[3]); |
| |
| /* Zero-masked. */ |
| |
| zmm_res.zmmi = _mm512_set1_epi32(-7); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc1); |
| zmm_res.zmmi = _mm512_maskz_load_epi32(k16, &isrc1.zmmi); |
| |
| zmm_res.zmmi = _mm512_set1_epi32(11); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc_arr[0]); |
| zmm_res.zmmi = _mm512_maskz_loadu_epi32(k16, &isrc_arr[0].s32[1]); |
| } |
| |
| void NOINLINE do_store_and_storeu_epi32() { |
| V512 xmm_res[2], ymm_res[2], zmm_res[2]; |
| __mmask16 k16 = 0xfeff; |
| __mmask8 k8 = (__mmask8)k16; |
| |
| /* Non-masked. */ |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(-101); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_store_epi32(&zmm_res[0].zmmi, isrc1.zmmi); |
| soft_v512_update(isrc1); |
| _mm256_store_si256((__m256i *)&ymm_res[0].s32[0], isrc1.ymmi[0]); |
| soft_v512_update(isrc1); |
| _mm_store_si128((__m128i *)&xmm_res[0].s32[0], isrc1.xmmi[0]); |
| |
| check_equal_nd(&ymm_res, &zmm_res, 8, "_mm256_store_epi32", __LINE__); |
| check_equal_nd(&xmm_res, &zmm_res, 4, "_mm_store_epi32", __LINE__); |
| |
| soft_v512_update(isrc1); |
| ymm_res[0].zmmi = _mm512_setzero_si512(); |
| _mm512_store_si512(&ymm_res[0].zmmi, isrc1.zmmi); |
| check_equal_nd(&ymm_res, &zmm_res, 16, "_mm512_store_si512", __LINE__); |
| |
| soft_v512_update(isrc1); |
| _mm512_storeu_si512(&zmm_res[0].s32[1], isrc1.zmmi); |
| soft_v512_update(isrc1); |
| _mm256_storeu_si256((__m256i *)&ymm_res[0].s32[1], isrc1.ymmi[0]); |
| soft_v512_update(isrc1); |
| _mm_storeu_si128((__m128i *)&xmm_res[0].s32[1], isrc1.xmmi[0]); |
| |
| check_equal_nd(&ymm_res[0].s32[1], &zmm_res[0].s32[1], 8, |
| "_mm256_storeu_si256", __LINE__); |
| check_equal_nd(&xmm_res[0].s32[1], &zmm_res[0].s32[1], 4, "_mm_storeu_si128", |
| __LINE__); |
| |
| /* Masked. */ |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(999); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_mask_store_epi32(&zmm_res[0].zmmi, k16, isrc1.zmmi); |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(-3); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_mask_storeu_epi32(&zmm_res[0].s32[1], k16, isrc1.zmmi); |
| } |
| |
| void NOINLINE do_load_and_loadu_epi64() { |
| V512 xmm_res, ymm_res, zmm_res; |
| __mmask16 k8 = 0xef; |
| |
| /* Non-masked. */ |
| |
| soft_v512_update(isrc1); |
| zmm_res.zmmi = _mm512_load_epi64(&isrc1.zmmi); |
| |
| soft_v512_update(isrc_arr[0]); |
| zmm_res.zmmi = _mm512_loadu_si512(&isrc_arr[0].s64[1]); |
| |
| /* Masked. */ |
| |
| zmm_res.zmmi = _mm512_setzero_epi32(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc1); |
| zmm_res.zmmi = _mm512_mask_load_epi64(zmm_res.zmmi, k8, &isrc1.zmmi); |
| |
| zmm_res.zmmi = _mm512_setzero_epi32(); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| |
| soft_v512_update(isrc_arr[0]); |
| zmm_res.zmmi = _mm512_mask_loadu_epi64(zmm_res.zmmi, k8, &isrc_arr[0].s64[3]); |
| |
| /* Zero-masked. */ |
| |
| zmm_res.zmmi = _mm512_set1_epi64(-7); |
| ymm_res = zmm_res; |
| xmm_res = zmm_res; |
| } |
| |
| void NOINLINE do_store_and_storeu_epi64() { |
| V512 xmm_res[2], ymm_res[2], zmm_res[2]; |
| __mmask8 k8 = 0xfe; |
| |
| /* Non-masked. */ |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(-101); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_store_epi64(&zmm_res[0].zmmi, isrc1.zmmi); |
| soft_v512_update(isrc1); |
| _mm256_store_si256((__m256i *)&ymm_res[0].s64[0], isrc1.ymmi[0]); |
| soft_v512_update(isrc1); |
| _mm_store_si128((__m128i *)&xmm_res[0].s64[0], isrc1.xmmi[0]); |
| |
| check_equal_nq(&ymm_res, &zmm_res, 4, "_mm256_store_epi64", __LINE__); |
| check_equal_nq(&xmm_res, &zmm_res, 2, "_mm_store_si256", __LINE__); |
| |
| soft_v512_update(isrc1); |
| _mm512_storeu_si512(&zmm_res[0].s64[1], isrc1.zmmi); |
| |
| /* Masked. */ |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(999); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_mask_store_epi64(&zmm_res[0].zmmi, k8, isrc1.zmmi); |
| |
| zmm_res[0].zmmi = _mm512_set1_epi32(-3); |
| zmm_res[1] = zmm_res[0]; |
| ymm_res[0] = zmm_res[0]; |
| ymm_res[1] = zmm_res[0]; |
| xmm_res[0] = zmm_res[0]; |
| xmm_res[1] = zmm_res[0]; |
| |
| soft_v512_update(isrc1); |
| _mm512_mask_storeu_epi64(&zmm_res[0].s64[1], k8, isrc1.zmmi); |
| } |
| |
| int main() { |
| init(); |
| |
| do_load_and_loadu_pd(); |
| do_load_and_loadu_ps(); |
| do_load_and_loadu_epi32(); |
| do_store_and_storeu_epi32(); |
| do_load_and_loadu_epi64(); |
| |
| do_store_and_storeu_pd(); |
| do_store_and_storeu_ps(); |
| |
| do_store_and_storeu_epi64(); |
| |
| if (n_errs != 0) { |
| printf("FAILED\n"); |
| return 1; |
| } |
| |
| printf("PASSED\n"); |
| return 0; |
| } |
