| //! Emulate x86 LLVM intrinsics |
| |
| use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; |
| use rustc_target::asm::*; |
| |
| use crate::inline_asm::{codegen_inline_asm_inner, CInlineAsmOperand}; |
| use crate::intrinsics::*; |
| use crate::prelude::*; |
| |
| pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( |
| fx: &mut FunctionCx<'_, '_, 'tcx>, |
| intrinsic: &str, |
| _args: GenericArgsRef<'tcx>, |
| args: &[Spanned<mir::Operand<'tcx>>], |
| ret: CPlace<'tcx>, |
| target: Option<BasicBlock>, |
| span: Span, |
| ) { |
| match intrinsic { |
| "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => { |
| // Spin loop hint |
| } |
| |
| "llvm.x86.avx.vzeroupper" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_zeroupper&ig_expand=7218 |
| // Do nothing. It is a perf hint anyway. |
| } |
| |
| // Used by is_x86_feature_detected!(); |
| "llvm.x86.xgetbv" => { |
| intrinsic_args!(fx, args => (xcr_no); intrinsic); |
| |
| let xcr_no = xcr_no.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String( |
| " |
| xgetbv |
| // out = rdx << 32 | rax |
| shl rdx, 32 |
| or rax, rdx |
| " |
| .to_string(), |
| )], |
| &[ |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), |
| value: xcr_no, |
| }, |
| CInlineAsmOperand::Out { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), |
| late: true, |
| place: Some(ret), |
| }, |
| CInlineAsmOperand::Out { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), |
| late: true, |
| place: None, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sse3.ldu.dq" | "llvm.x86.avx.ldu.dq.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_lddqu_si128&ig_expand=4009 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_lddqu_si256&ig_expand=4010 |
| intrinsic_args!(fx, args => (ptr); intrinsic); |
| |
| // FIXME correctly handle unalignedness |
| let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), ret.layout()); |
| ret.write_cvalue(fx, val); |
| } |
| |
| "llvm.x86.avx2.gather.d.d" |
| | "llvm.x86.avx2.gather.d.q" |
| | "llvm.x86.avx2.gather.d.ps" |
| | "llvm.x86.avx2.gather.d.pd" |
| | "llvm.x86.avx2.gather.d.d.256" |
| | "llvm.x86.avx2.gather.d.q.256" |
| | "llvm.x86.avx2.gather.d.ps.256" |
| | "llvm.x86.avx2.gather.d.pd.256" |
| | "llvm.x86.avx2.gather.q.d" |
| | "llvm.x86.avx2.gather.q.q" |
| | "llvm.x86.avx2.gather.q.ps" |
| | "llvm.x86.avx2.gather.q.pd" |
| | "llvm.x86.avx2.gather.q.d.256" |
| | "llvm.x86.avx2.gather.q.q.256" |
| | "llvm.x86.avx2.gather.q.ps.256" |
| | "llvm.x86.avx2.gather.q.pd.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_pd&ig_expand=3818 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_pd&ig_expand=3819 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_pd&ig_expand=3821 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_pd&ig_expand=3822 |
| // ... |
| |
| intrinsic_args!(fx, args => (src, ptr, index, mask, scale); intrinsic); |
| |
| let (src_lane_count, src_lane_ty) = src.layout().ty.simd_size_and_type(fx.tcx); |
| let (index_lane_count, index_lane_ty) = index.layout().ty.simd_size_and_type(fx.tcx); |
| let (mask_lane_count, mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(src_lane_ty, ret_lane_ty); |
| assert!(index_lane_ty.is_integral()); |
| assert_eq!(src_lane_count, mask_lane_count); |
| assert_eq!(src_lane_count, ret_lane_count); |
| |
| let lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap(); |
| let index_lane_clif_ty = fx.clif_type(index_lane_ty).unwrap(); |
| let mask_lane_clif_ty = fx.clif_type(mask_lane_ty).unwrap(); |
| let ret_lane_layout = fx.layout_of(ret_lane_ty); |
| |
| let ptr = ptr.load_scalar(fx); |
| let scale = scale.load_scalar(fx); |
| let scale = fx.bcx.ins().uextend(types::I64, scale); |
| for lane_idx in 0..std::cmp::min(src_lane_count, index_lane_count) { |
| let src_lane = src.value_lane(fx, lane_idx).load_scalar(fx); |
| let index_lane = index.value_lane(fx, lane_idx).load_scalar(fx); |
| let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); |
| let mask_lane = |
| fx.bcx.ins().bitcast(mask_lane_clif_ty.as_int(), MemFlags::new(), mask_lane); |
| |
| let if_enabled = fx.bcx.create_block(); |
| let if_disabled = fx.bcx.create_block(); |
| let next = fx.bcx.create_block(); |
| let res_lane = fx.bcx.append_block_param(next, lane_clif_ty); |
| |
| let mask_lane = match mask_lane_clif_ty { |
| types::I32 | types::F32 => { |
| fx.bcx.ins().band_imm(mask_lane, 0x8000_0000u64 as i64) |
| } |
| types::I64 | types::F64 => { |
| fx.bcx.ins().band_imm(mask_lane, 0x8000_0000_0000_0000u64 as i64) |
| } |
| _ => unreachable!(), |
| }; |
| fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]); |
| fx.bcx.seal_block(if_enabled); |
| fx.bcx.seal_block(if_disabled); |
| |
| fx.bcx.switch_to_block(if_enabled); |
| let index_lane = if index_lane_clif_ty != types::I64 { |
| fx.bcx.ins().sextend(types::I64, index_lane) |
| } else { |
| index_lane |
| }; |
| let offset = fx.bcx.ins().imul(index_lane, scale); |
| let lane_ptr = fx.bcx.ins().iadd(ptr, offset); |
| let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0); |
| fx.bcx.ins().jump(next, &[res]); |
| |
| fx.bcx.switch_to_block(if_disabled); |
| fx.bcx.ins().jump(next, &[src_lane]); |
| |
| fx.bcx.seal_block(next); |
| fx.bcx.switch_to_block(next); |
| |
| fx.bcx.ins().nop(); |
| |
| ret.place_lane(fx, lane_idx) |
| .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout)); |
| } |
| |
| for lane_idx in std::cmp::min(src_lane_count, index_lane_count)..ret_lane_count { |
| let zero_lane = fx.bcx.ins().iconst(mask_lane_clif_ty.as_int(), 0); |
| let zero_lane = fx.bcx.ins().bitcast(mask_lane_clif_ty, MemFlags::new(), zero_lane); |
| ret.place_lane(fx, lane_idx) |
| .write_cvalue(fx, CValue::by_val(zero_lane, ret_lane_layout)); |
| } |
| } |
| |
| "llvm.x86.sse.add.ss" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ss&ig_expand=171 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| assert_eq!(a.layout(), ret.layout()); |
| let layout = a.layout(); |
| |
| let (_, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| assert!(lane_ty.is_floating_point()); |
| let ret_lane_layout = fx.layout_of(lane_ty); |
| |
| ret.write_cvalue(fx, a); |
| |
| let a_lane = a.value_lane(fx, 0).load_scalar(fx); |
| let b_lane = b.value_lane(fx, 0).load_scalar(fx); |
| |
| let res = fx.bcx.ins().fadd(a_lane, b_lane); |
| |
| let res_lane = CValue::by_val(res, ret_lane_layout); |
| ret.place_lane(fx, 0).write_cvalue(fx, res_lane); |
| } |
| |
| "llvm.x86.sse.sqrt.ps" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ps&ig_expand=6245 |
| intrinsic_args!(fx, args => (a); intrinsic); |
| |
| // FIXME use vector instructions when possible |
| simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { |
| fx.bcx.ins().sqrt(lane) |
| }); |
| } |
| |
| "llvm.x86.sse.max.ps" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps&ig_expand=4357 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| simd_pair_for_each_lane( |
| fx, |
| a, |
| b, |
| ret, |
| &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmax(a_lane, b_lane), |
| ); |
| } |
| |
| "llvm.x86.sse.min.ps" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps&ig_expand=4489 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| simd_pair_for_each_lane( |
| fx, |
| a, |
| b, |
| ret, |
| &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmin(a_lane, b_lane), |
| ); |
| } |
| |
| "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { |
| let (x, y, kind) = match args { |
| [x, y, kind] => (x, y, kind), |
| _ => bug!("wrong number of args for intrinsic {intrinsic}"), |
| }; |
| let x = codegen_operand(fx, &x.node); |
| let y = codegen_operand(fx, &y.node); |
| let kind = match &kind.node { |
| Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0, |
| Operand::Copy(_) | Operand::Move(_) => unreachable!("{kind:?}"), |
| }; |
| |
| let flt_cc = match kind |
| .try_to_bits(Size::from_bytes(1)) |
| .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) |
| .try_into() |
| .unwrap() |
| { |
| _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal, |
| _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan, |
| _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual, |
| _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered, |
| _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual, |
| _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual, |
| _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan, |
| _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered, |
| _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual, |
| _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan, |
| _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual, |
| _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(), |
| _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual, |
| _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual, |
| _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan, |
| _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(), |
| |
| kind => unreachable!("kind {:?}", kind), |
| }; |
| |
| // Copied from stdarch |
| /// Equal (ordered, non-signaling) |
| const _CMP_EQ_OQ: i32 = 0x00; |
| /// Less-than (ordered, signaling) |
| const _CMP_LT_OS: i32 = 0x01; |
| /// Less-than-or-equal (ordered, signaling) |
| const _CMP_LE_OS: i32 = 0x02; |
| /// Unordered (non-signaling) |
| const _CMP_UNORD_Q: i32 = 0x03; |
| /// Not-equal (unordered, non-signaling) |
| const _CMP_NEQ_UQ: i32 = 0x04; |
| /// Not-less-than (unordered, signaling) |
| const _CMP_NLT_US: i32 = 0x05; |
| /// Not-less-than-or-equal (unordered, signaling) |
| const _CMP_NLE_US: i32 = 0x06; |
| /// Ordered (non-signaling) |
| const _CMP_ORD_Q: i32 = 0x07; |
| /// Equal (unordered, non-signaling) |
| const _CMP_EQ_UQ: i32 = 0x08; |
| /// Not-greater-than-or-equal (unordered, signaling) |
| const _CMP_NGE_US: i32 = 0x09; |
| /// Not-greater-than (unordered, signaling) |
| const _CMP_NGT_US: i32 = 0x0a; |
| /// False (ordered, non-signaling) |
| const _CMP_FALSE_OQ: i32 = 0x0b; |
| /// Not-equal (ordered, non-signaling) |
| const _CMP_NEQ_OQ: i32 = 0x0c; |
| /// Greater-than-or-equal (ordered, signaling) |
| const _CMP_GE_OS: i32 = 0x0d; |
| /// Greater-than (ordered, signaling) |
| const _CMP_GT_OS: i32 = 0x0e; |
| /// True (unordered, non-signaling) |
| const _CMP_TRUE_UQ: i32 = 0x0f; |
| /// Equal (ordered, signaling) |
| const _CMP_EQ_OS: i32 = 0x10; |
| /// Less-than (ordered, non-signaling) |
| const _CMP_LT_OQ: i32 = 0x11; |
| /// Less-than-or-equal (ordered, non-signaling) |
| const _CMP_LE_OQ: i32 = 0x12; |
| /// Unordered (signaling) |
| const _CMP_UNORD_S: i32 = 0x13; |
| /// Not-equal (unordered, signaling) |
| const _CMP_NEQ_US: i32 = 0x14; |
| /// Not-less-than (unordered, non-signaling) |
| const _CMP_NLT_UQ: i32 = 0x15; |
| /// Not-less-than-or-equal (unordered, non-signaling) |
| const _CMP_NLE_UQ: i32 = 0x16; |
| /// Ordered (signaling) |
| const _CMP_ORD_S: i32 = 0x17; |
| /// Equal (unordered, signaling) |
| const _CMP_EQ_US: i32 = 0x18; |
| /// Not-greater-than-or-equal (unordered, non-signaling) |
| const _CMP_NGE_UQ: i32 = 0x19; |
| /// Not-greater-than (unordered, non-signaling) |
| const _CMP_NGT_UQ: i32 = 0x1a; |
| /// False (ordered, signaling) |
| const _CMP_FALSE_OS: i32 = 0x1b; |
| /// Not-equal (ordered, signaling) |
| const _CMP_NEQ_OS: i32 = 0x1c; |
| /// Greater-than-or-equal (ordered, non-signaling) |
| const _CMP_GE_OQ: i32 = 0x1d; |
| /// Greater-than (ordered, non-signaling) |
| const _CMP_GT_OQ: i32 = 0x1e; |
| /// True (unordered, signaling) |
| const _CMP_TRUE_US: i32 = 0x1f; |
| |
| simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { |
| let res_lane = match lane_ty.kind() { |
| ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane), |
| _ => unreachable!("{:?}", lane_ty), |
| }; |
| bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) |
| }); |
| } |
| "llvm.x86.ssse3.pshuf.b.128" | "llvm.x86.avx2.pshuf.b" => { |
| let (a, b) = match args { |
| [a, b] => (a, b), |
| _ => bug!("wrong number of args for intrinsic {intrinsic}"), |
| }; |
| let a = codegen_operand(fx, &a.node); |
| let b = codegen_operand(fx, &b.node); |
| |
| // Based on the pseudocode at https://github.com/rust-lang/stdarch/blob/1cfbca8b38fd9b4282b2f054f61c6ca69fc7ce29/crates/core_arch/src/x86/avx2.rs#L2319-L2332 |
| let zero = fx.bcx.ins().iconst(types::I8, 0); |
| for i in 0..16 { |
| let b_lane = b.value_lane(fx, i).load_scalar(fx); |
| let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); |
| let a_idx = fx.bcx.ins().band_imm(b_lane, 0xf); |
| let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); |
| let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); |
| let res = fx.bcx.ins().select(is_zero, zero, a_lane); |
| ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); |
| } |
| |
| if intrinsic == "llvm.x86.avx2.pshuf.b" { |
| for i in 16..32 { |
| let b_lane = b.value_lane(fx, i).load_scalar(fx); |
| let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); |
| let b_lane_masked = fx.bcx.ins().band_imm(b_lane, 0xf); |
| let a_idx = fx.bcx.ins().iadd_imm(b_lane_masked, 16); |
| let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); |
| let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); |
| let res = fx.bcx.ins().select(is_zero, zero, a_lane); |
| ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); |
| } |
| } |
| } |
| "llvm.x86.avx2.permd" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permutevar8x32_epi32 |
| intrinsic_args!(fx, args => (a, idx); intrinsic); |
| |
| for j in 0..=7 { |
| let index = idx.value_typed_lane(fx, fx.tcx.types.u32, j).load_scalar(fx); |
| let index = fx.bcx.ins().uextend(fx.pointer_type, index); |
| let value = a.value_lane_dyn(fx, index).load_scalar(fx); |
| ret.place_typed_lane(fx, fx.tcx.types.u32, j).to_ptr().store( |
| fx, |
| value, |
| MemFlags::trusted(), |
| ); |
| } |
| } |
| "llvm.x86.avx2.vperm2i128" |
| | "llvm.x86.avx.vperm2f128.ps.256" |
| | "llvm.x86.avx.vperm2f128.pd.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2x128_si256 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2f128_ps |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2f128_pd |
| let (a, b, imm8) = match args { |
| [a, b, imm8] => (a, b, imm8), |
| _ => bug!("wrong number of args for intrinsic {intrinsic}"), |
| }; |
| let a = codegen_operand(fx, &a.node); |
| let b = codegen_operand(fx, &b.node); |
| let imm8 = codegen_operand(fx, &imm8.node).load_scalar(fx); |
| |
| let a_low = a.value_typed_lane(fx, fx.tcx.types.u128, 0).load_scalar(fx); |
| let a_high = a.value_typed_lane(fx, fx.tcx.types.u128, 1).load_scalar(fx); |
| |
| let b_low = b.value_typed_lane(fx, fx.tcx.types.u128, 0).load_scalar(fx); |
| let b_high = b.value_typed_lane(fx, fx.tcx.types.u128, 1).load_scalar(fx); |
| |
| fn select4( |
| fx: &mut FunctionCx<'_, '_, '_>, |
| a_high: Value, |
| a_low: Value, |
| b_high: Value, |
| b_low: Value, |
| control: Value, |
| ) -> Value { |
| let a_or_b = fx.bcx.ins().band_imm(control, 0b0010); |
| let high_or_low = fx.bcx.ins().band_imm(control, 0b0001); |
| let is_zero = fx.bcx.ins().band_imm(control, 0b1000); |
| |
| let zero = fx.bcx.ins().iconst(types::I64, 0); |
| let zero = fx.bcx.ins().iconcat(zero, zero); |
| |
| let res_a = fx.bcx.ins().select(high_or_low, a_high, a_low); |
| let res_b = fx.bcx.ins().select(high_or_low, b_high, b_low); |
| let res = fx.bcx.ins().select(a_or_b, res_b, res_a); |
| fx.bcx.ins().select(is_zero, zero, res) |
| } |
| |
| let control0 = imm8; |
| let res_low = select4(fx, a_high, a_low, b_high, b_low, control0); |
| |
| let control1 = fx.bcx.ins().ushr_imm(imm8, 4); |
| let res_high = select4(fx, a_high, a_low, b_high, b_low, control1); |
| |
| ret.place_typed_lane(fx, fx.tcx.types.u128, 0).to_ptr().store( |
| fx, |
| res_low, |
| MemFlags::trusted(), |
| ); |
| ret.place_typed_lane(fx, fx.tcx.types.u128, 1).to_ptr().store( |
| fx, |
| res_high, |
| MemFlags::trusted(), |
| ); |
| } |
| "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { |
| intrinsic_args!(fx, args => (a); intrinsic); |
| |
| simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { |
| fx.bcx.ins().iabs(lane) |
| }); |
| } |
| "llvm.x86.sse2.cvttps2dq" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttps_epi32&ig_expand=2429 |
| intrinsic_args!(fx, args => (a); intrinsic); |
| let a = a.load_scalar(fx); |
| |
| // Using inline asm instead of fcvt_to_sint_sat as unrepresentable values are turned |
| // into 0x80000000 for which Cranelift doesn't have a native instruction. |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("cvttps2dq xmm0, xmm0"))], |
| &[CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => { |
| intrinsic_args!(fx, args => (c_in, a, b); intrinsic); |
| let c_in = c_in.load_scalar(fx); |
| |
| let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); |
| |
| let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); |
| let val = CValue::by_val_pair(cb_out, c, layout); |
| ret.write_cvalue(fx, val); |
| } |
| "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => { |
| intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic); |
| let c_in = c_in.load_scalar(fx); |
| |
| let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); |
| |
| Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted()); |
| ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8))); |
| } |
| "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => { |
| intrinsic_args!(fx, args => (b_in, a, b); intrinsic); |
| let b_in = b_in.load_scalar(fx); |
| |
| let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b); |
| |
| let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); |
| let val = CValue::by_val_pair(cb_out, c, layout); |
| ret.write_cvalue(fx, val); |
| } |
| "llvm.x86.sse2.pavg.b" | "llvm.x86.sse2.pavg.w" => { |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| // FIXME use vector instructions when possible |
| simd_pair_for_each_lane( |
| fx, |
| a, |
| b, |
| ret, |
| &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { |
| // (a + b + 1) >> 1 |
| let lane_ty = fx.bcx.func.dfg.value_type(a_lane); |
| let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane); |
| let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane); |
| let sum = fx.bcx.ins().iadd(a_lane, b_lane); |
| let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1); |
| let res = fx.bcx.ins().ushr_imm(num_plus_one, 1); |
| fx.bcx.ins().ireduce(lane_ty, res) |
| }, |
| ); |
| } |
| "llvm.x86.sse2.psra.w" => { |
| intrinsic_args!(fx, args => (a, count); intrinsic); |
| |
| let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted()); |
| let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap(); |
| let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1)); |
| let saturated_count = fx.bcx.ins().umin(count_lane, max_count); |
| |
| // FIXME use vector instructions when possible |
| simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, a_lane| { |
| fx.bcx.ins().sshr(a_lane, saturated_count) |
| }); |
| } |
| "llvm.x86.sse2.psad.bw" | "llvm.x86.avx2.psad.bw" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sad_epu8&ig_expand=5770 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sad_epu8&ig_expand=5771 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(lane_ty, fx.tcx.types.u8); |
| assert_eq!(ret_lane_ty, fx.tcx.types.u64); |
| assert_eq!(lane_count, ret_lane_count * 8); |
| |
| let ret_lane_layout = fx.layout_of(fx.tcx.types.u64); |
| for out_lane_idx in 0..lane_count / 8 { |
| let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0); |
| |
| for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 8 { |
| let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx); |
| let a_lane = fx.bcx.ins().uextend(types::I16, a_lane); |
| let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx); |
| let b_lane = fx.bcx.ins().uextend(types::I16, b_lane); |
| |
| let lane_diff = fx.bcx.ins().isub(a_lane, b_lane); |
| let abs_lane_diff = fx.bcx.ins().iabs(lane_diff); |
| let abs_lane_diff = fx.bcx.ins().uextend(types::I64, abs_lane_diff); |
| lane_diff_acc = fx.bcx.ins().iadd(lane_diff_acc, abs_lane_diff); |
| } |
| |
| let res_lane = CValue::by_val(lane_diff_acc, ret_lane_layout); |
| |
| ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); |
| } |
| } |
| "llvm.x86.ssse3.pmadd.ub.sw.128" | "llvm.x86.avx2.pmadd.ub.sw" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maddubs_epi16&ig_expand=4267 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maddubs_epi16&ig_expand=4270 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(lane_ty, fx.tcx.types.u8); |
| assert_eq!(ret_lane_ty, fx.tcx.types.i16); |
| assert_eq!(lane_count, ret_lane_count * 2); |
| |
| let ret_lane_layout = fx.layout_of(fx.tcx.types.i16); |
| for out_lane_idx in 0..lane_count / 2 { |
| let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx); |
| let a_lane0 = fx.bcx.ins().uextend(types::I16, a_lane0); |
| let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx); |
| let b_lane0 = fx.bcx.ins().sextend(types::I16, b_lane0); |
| |
| let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); |
| let a_lane1 = fx.bcx.ins().uextend(types::I16, a_lane1); |
| let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); |
| let b_lane1 = fx.bcx.ins().sextend(types::I16, b_lane1); |
| |
| let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0); |
| let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1); |
| |
| let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1); |
| |
| let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0); |
| |
| let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16)); |
| let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16)); |
| |
| let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); |
| let res_lane = fx.bcx.ins().select(has_overflow, sat_val, val); |
| |
| let res_lane = CValue::by_val(res_lane, ret_lane_layout); |
| |
| ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); |
| } |
| } |
| "llvm.x86.sse2.pmadd.wd" | "llvm.x86.avx2.pmadd.wd" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_madd_epi16&ig_expand=4231 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_madd_epi16&ig_expand=4234 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(lane_ty, fx.tcx.types.i16); |
| assert_eq!(ret_lane_ty, fx.tcx.types.i32); |
| assert_eq!(lane_count, ret_lane_count * 2); |
| |
| let ret_lane_layout = fx.layout_of(fx.tcx.types.i32); |
| for out_lane_idx in 0..lane_count / 2 { |
| let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx); |
| let a_lane0 = fx.bcx.ins().sextend(types::I32, a_lane0); |
| let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx); |
| let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0); |
| |
| let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); |
| let a_lane1 = fx.bcx.ins().sextend(types::I32, a_lane1); |
| let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); |
| let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1); |
| |
| let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0); |
| let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1); |
| |
| let res_lane = fx.bcx.ins().iadd(mul0, mul1); |
| let res_lane = CValue::by_val(res_lane, ret_lane_layout); |
| |
| ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); |
| } |
| } |
| |
| "llvm.x86.ssse3.pmul.hr.sw.128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhrs_epi16&ig_expand=4782 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(lane_ty, fx.tcx.types.i16); |
| assert_eq!(ret_lane_ty, fx.tcx.types.i16); |
| assert_eq!(lane_count, ret_lane_count); |
| |
| let ret_lane_layout = fx.layout_of(fx.tcx.types.i16); |
| for out_lane_idx in 0..lane_count { |
| let a_lane = a.value_lane(fx, out_lane_idx).load_scalar(fx); |
| let a_lane = fx.bcx.ins().sextend(types::I32, a_lane); |
| let b_lane = b.value_lane(fx, out_lane_idx).load_scalar(fx); |
| let b_lane = fx.bcx.ins().sextend(types::I32, b_lane); |
| |
| let mul: Value = fx.bcx.ins().imul(a_lane, b_lane); |
| let shifted = fx.bcx.ins().ushr_imm(mul, 14); |
| let incremented = fx.bcx.ins().iadd_imm(shifted, 1); |
| let shifted_again = fx.bcx.ins().ushr_imm(incremented, 1); |
| |
| let res_lane = fx.bcx.ins().ireduce(types::I16, shifted_again); |
| let res_lane = CValue::by_val(res_lane, ret_lane_layout); |
| |
| ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); |
| } |
| } |
| |
| "llvm.x86.sse2.packuswb.128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi16&ig_expand=4903 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::U8, PackWidth::Sse); |
| } |
| |
| "llvm.x86.sse2.packsswb.128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi16&ig_expand=4848 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::S8, PackWidth::Sse); |
| } |
| |
| "llvm.x86.avx2.packuswb" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi16&ig_expand=4906 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::U8, PackWidth::Avx); |
| } |
| |
| "llvm.x86.avx2.packsswb" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi16&ig_expand=4851 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::S8, PackWidth::Avx); |
| } |
| |
| "llvm.x86.sse41.packusdw" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi32&ig_expand=4912 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::U16, PackWidth::Sse); |
| } |
| |
| "llvm.x86.sse2.packssdw.128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi32&ig_expand=4889 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::S16, PackWidth::Sse); |
| } |
| |
| "llvm.x86.avx2.packusdw" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi32&ig_expand=4883 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::U16, PackWidth::Avx); |
| } |
| |
| "llvm.x86.avx2.packssdw" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi32&ig_expand=4892 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| pack_instruction(fx, a, b, ret, PackSize::S16, PackWidth::Avx); |
| } |
| |
| "llvm.x86.fma.vfmaddsub.ps" |
| | "llvm.x86.fma.vfmaddsub.pd" |
| | "llvm.x86.fma.vfmaddsub.ps.256" |
| | "llvm.x86.fma.vfmaddsub.pd.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmaddsub_ps&ig_expand=3205 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmaddsub_pd&ig_expand=3181 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmaddsub_ps&ig_expand=3209 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmaddsub_pd&ig_expand=3185 |
| intrinsic_args!(fx, args => (a, b, c); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| assert_eq!(a.layout(), c.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert!(lane_ty.is_floating_point()); |
| assert!(ret_lane_ty.is_floating_point()); |
| assert_eq!(lane_count, ret_lane_count); |
| let ret_lane_layout = fx.layout_of(ret_lane_ty); |
| |
| for idx in 0..lane_count { |
| let a_lane = a.value_lane(fx, idx).load_scalar(fx); |
| let b_lane = b.value_lane(fx, idx).load_scalar(fx); |
| let c_lane = c.value_lane(fx, idx).load_scalar(fx); |
| |
| let mul = fx.bcx.ins().fmul(a_lane, b_lane); |
| let res = if idx & 1 == 0 { |
| fx.bcx.ins().fsub(mul, c_lane) |
| } else { |
| fx.bcx.ins().fadd(mul, c_lane) |
| }; |
| |
| let res_lane = CValue::by_val(res, ret_lane_layout); |
| ret.place_lane(fx, idx).write_cvalue(fx, res_lane); |
| } |
| } |
| |
| "llvm.x86.fma.vfmsubadd.ps" |
| | "llvm.x86.fma.vfmsubadd.pd" |
| | "llvm.x86.fma.vfmsubadd.ps.256" |
| | "llvm.x86.fma.vfmsubadd.pd.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmsubadd_ps&ig_expand=3325 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmsubadd_pd&ig_expand=3301 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmsubadd_ps&ig_expand=3329 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmsubadd_pd&ig_expand=3305 |
| intrinsic_args!(fx, args => (a, b, c); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| assert_eq!(a.layout(), c.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert!(lane_ty.is_floating_point()); |
| assert!(ret_lane_ty.is_floating_point()); |
| assert_eq!(lane_count, ret_lane_count); |
| let ret_lane_layout = fx.layout_of(ret_lane_ty); |
| |
| for idx in 0..lane_count { |
| let a_lane = a.value_lane(fx, idx).load_scalar(fx); |
| let b_lane = b.value_lane(fx, idx).load_scalar(fx); |
| let c_lane = c.value_lane(fx, idx).load_scalar(fx); |
| |
| let mul = fx.bcx.ins().fmul(a_lane, b_lane); |
| let res = if idx & 1 == 0 { |
| fx.bcx.ins().fadd(mul, c_lane) |
| } else { |
| fx.bcx.ins().fsub(mul, c_lane) |
| }; |
| |
| let res_lane = CValue::by_val(res, ret_lane_layout); |
| ret.place_lane(fx, idx).write_cvalue(fx, res_lane); |
| } |
| } |
| |
| "llvm.x86.fma.vfnmadd.ps" |
| | "llvm.x86.fma.vfnmadd.pd" |
| | "llvm.x86.fma.vfnmadd.ps.256" |
| | "llvm.x86.fma.vfnmadd.pd.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fnmadd_ps&ig_expand=3391 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fnmadd_pd&ig_expand=3367 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fnmadd_ps&ig_expand=3395 |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fnmadd_pd&ig_expand=3371 |
| intrinsic_args!(fx, args => (a, b, c); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| assert_eq!(a.layout(), c.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert!(lane_ty.is_floating_point()); |
| assert!(ret_lane_ty.is_floating_point()); |
| assert_eq!(lane_count, ret_lane_count); |
| let ret_lane_layout = fx.layout_of(ret_lane_ty); |
| |
| for idx in 0..lane_count { |
| let a_lane = a.value_lane(fx, idx).load_scalar(fx); |
| let b_lane = b.value_lane(fx, idx).load_scalar(fx); |
| let c_lane = c.value_lane(fx, idx).load_scalar(fx); |
| |
| let mul = fx.bcx.ins().fmul(a_lane, b_lane); |
| let neg_mul = fx.bcx.ins().fneg(mul); |
| let res = fx.bcx.ins().fadd(neg_mul, c_lane); |
| |
| let res_lane = CValue::by_val(res, ret_lane_layout); |
| ret.place_lane(fx, idx).write_cvalue(fx, res_lane); |
| } |
| } |
| |
| "llvm.x86.sse42.crc32.32.8" |
| | "llvm.x86.sse42.crc32.32.16" |
| | "llvm.x86.sse42.crc32.32.32" |
| | "llvm.x86.sse42.crc32.64.64" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#ig_expand=1419&text=_mm_crc32_u32 |
| intrinsic_args!(fx, args => (crc, v); intrinsic); |
| |
| let crc = crc.load_scalar(fx); |
| let v = v.load_scalar(fx); |
| |
| let asm = match intrinsic { |
| "llvm.x86.sse42.crc32.32.8" => "crc32 eax, dl", |
| "llvm.x86.sse42.crc32.32.16" => "crc32 eax, dx", |
| "llvm.x86.sse42.crc32.32.32" => "crc32 eax, edx", |
| "llvm.x86.sse42.crc32.64.64" => "crc32 rax, rdx", |
| _ => unreachable!(), |
| }; |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(asm.to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), |
| _late: true, |
| in_value: crc, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), |
| value: v, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sse42.pcmpestri128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestri&ig_expand=939 |
| intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let la = la.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| let lb = lb.load_scalar(fx); |
| |
| let imm8 = |
| if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4].node) { |
| imm8 |
| } else { |
| fx.tcx |
| .dcx() |
| .span_fatal(span, "Index argument for `_mm_cmpestri` is not a constant"); |
| }; |
| |
| let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8)); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("pcmpestri xmm0, xmm1, {imm8}"))], |
| &[ |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| value: a, |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: b, |
| }, |
| // Implicit argument to the pcmpestri intrinsic |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), |
| value: la, |
| }, |
| // Implicit argument to the pcmpestri intrinsic |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), |
| value: lb, |
| }, |
| // Implicit result of the pcmpestri intrinsic |
| CInlineAsmOperand::Out { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), |
| late: true, |
| place: Some(ret), |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sse42.pcmpestrm128" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrm&ig_expand=940 |
| intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let la = la.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| let lb = lb.load_scalar(fx); |
| |
| let imm8 = |
| if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4].node) { |
| imm8 |
| } else { |
| fx.tcx |
| .dcx() |
| .span_fatal(span, "Index argument for `_mm_cmpestrm` is not a constant"); |
| }; |
| |
| let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8)); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("pcmpestrm xmm0, xmm1, {imm8}"))], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: b, |
| }, |
| // Implicit argument to the pcmpestri intrinsic |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), |
| value: la, |
| }, |
| // Implicit argument to the pcmpestri intrinsic |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), |
| value: lb, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.pclmulqdq" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128&ig_expand=772 |
| intrinsic_args!(fx, args => (a, b, _imm8); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| let imm8 = |
| if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[2].node) { |
| imm8 |
| } else { |
| fx.tcx.dcx().span_fatal( |
| span, |
| "Index argument for `_mm_clmulepi64_si128` is not a constant", |
| ); |
| }; |
| |
| let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8)); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("pclmulqdq xmm0, xmm1, {imm8}"))], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aeskeygenassist" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aeskeygenassist_si128&ig_expand=261 |
| intrinsic_args!(fx, args => (a, _imm8); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| |
| let imm8 = |
| if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[1].node) { |
| imm8 |
| } else { |
| fx.tcx.dcx().span_fatal( |
| span, |
| "Index argument for `_mm_aeskeygenassist_si128` is not a constant", |
| ); |
| }; |
| |
| let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8)); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("aeskeygenassist xmm0, xmm0, {imm8}"))], |
| &[CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aesimc" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesimc_si128&ig_expand=260 |
| intrinsic_args!(fx, args => (a); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("aesimc xmm0, xmm0".to_string())], |
| &[CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aesenc" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc_si128&ig_expand=252 |
| intrinsic_args!(fx, args => (a, round_key); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let round_key = round_key.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("aesenc xmm0, xmm1".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: round_key, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aesenclast" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenclast_si128&ig_expand=257 |
| intrinsic_args!(fx, args => (a, round_key); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let round_key = round_key.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("aesenclast xmm0, xmm1".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: round_key, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aesdec" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec_si128&ig_expand=242 |
| intrinsic_args!(fx, args => (a, round_key); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let round_key = round_key.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("aesdec xmm0, xmm1".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: round_key, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.aesni.aesdeclast" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdeclast_si128&ig_expand=247 |
| intrinsic_args!(fx, args => (a, round_key); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let round_key = round_key.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("aesdeclast xmm0, xmm1".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| value: round_key, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha1rnds4" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1rnds4_epu32&ig_expand=5877 |
| intrinsic_args!(fx, args => (a, b, _func); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| let func = if let Some(func) = |
| crate::constant::mir_operand_get_const_val(fx, &args[2].node) |
| { |
| func |
| } else { |
| fx.tcx |
| .dcx() |
| .span_fatal(span, "Func argument for `_mm_sha1rnds4_epu32` is not a constant"); |
| }; |
| |
| let func = func.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", func)); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String(format!("sha1rnds4 xmm1, xmm2, {func}"))], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha1msg1" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg1_epu32&ig_expand=5874 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha1msg1 xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha1msg2" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg2_epu32&ig_expand=5875 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha1msg2 xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha1nexte" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1nexte_epu32&ig_expand=5876 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha1nexte xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha256rnds2" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256rnds2_epu32&ig_expand=5977 |
| intrinsic_args!(fx, args => (a, b, k); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| let k = k.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha256rnds2 xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| // Implicit argument to the sha256rnds2 instruction |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), |
| value: k, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha256msg1" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg1_epu32&ig_expand=5975 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha256msg1 xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.sha256msg2" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg2_epu32&ig_expand=5976 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| let a = a.load_scalar(fx); |
| let b = b.load_scalar(fx); |
| |
| codegen_inline_asm_inner( |
| fx, |
| &[InlineAsmTemplatePiece::String("sha256msg2 xmm1, xmm2".to_string())], |
| &[ |
| CInlineAsmOperand::InOut { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), |
| _late: true, |
| in_value: a, |
| out_place: Some(ret), |
| }, |
| CInlineAsmOperand::In { |
| reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), |
| value: b, |
| }, |
| ], |
| InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, |
| ); |
| } |
| |
| "llvm.x86.avx.ptestz.256" => { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_testz_si256&ig_expand=6945 |
| intrinsic_args!(fx, args => (a, b); intrinsic); |
| |
| assert_eq!(a.layout(), b.layout()); |
| let layout = a.layout(); |
| |
| let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| assert_eq!(lane_ty, fx.tcx.types.i64); |
| assert_eq!(ret.layout().ty, fx.tcx.types.i32); |
| assert_eq!(lane_count, 4); |
| |
| let a_lane0 = a.value_lane(fx, 0).load_scalar(fx); |
| let a_lane1 = a.value_lane(fx, 1).load_scalar(fx); |
| let a_lane2 = a.value_lane(fx, 2).load_scalar(fx); |
| let a_lane3 = a.value_lane(fx, 3).load_scalar(fx); |
| let b_lane0 = b.value_lane(fx, 0).load_scalar(fx); |
| let b_lane1 = b.value_lane(fx, 1).load_scalar(fx); |
| let b_lane2 = b.value_lane(fx, 2).load_scalar(fx); |
| let b_lane3 = b.value_lane(fx, 3).load_scalar(fx); |
| |
| let zero0 = fx.bcx.ins().band(a_lane0, b_lane0); |
| let zero1 = fx.bcx.ins().band(a_lane1, b_lane1); |
| let zero2 = fx.bcx.ins().band(a_lane2, b_lane2); |
| let zero3 = fx.bcx.ins().band(a_lane3, b_lane3); |
| |
| let all_zero0 = fx.bcx.ins().bor(zero0, zero1); |
| let all_zero1 = fx.bcx.ins().bor(zero2, zero3); |
| let all_zero = fx.bcx.ins().bor(all_zero0, all_zero1); |
| |
| let res = fx.bcx.ins().icmp_imm(IntCC::Equal, all_zero, 0); |
| let res = CValue::by_val( |
| fx.bcx.ins().uextend(types::I32, res), |
| fx.layout_of(fx.tcx.types.i32), |
| ); |
| ret.write_cvalue(fx, res); |
| } |
| |
| _ => { |
| fx.tcx |
| .dcx() |
| .warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic)); |
| crate::trap::trap_unimplemented(fx, intrinsic); |
| return; |
| } |
| } |
| |
| let dest = target.expect("all llvm intrinsics used by stdlib should return"); |
| let ret_block = fx.get_block(dest); |
| fx.bcx.ins().jump(ret_block, &[]); |
| } |
| |
| // llvm.x86.avx2.vperm2i128 |
| // llvm.x86.ssse3.pshuf.b.128 |
| // llvm.x86.avx2.pshuf.b |
| |
| fn llvm_add_sub<'tcx>( |
| fx: &mut FunctionCx<'_, '_, 'tcx>, |
| bin_op: BinOp, |
| cb_in: Value, |
| a: CValue<'tcx>, |
| b: CValue<'tcx>, |
| ) -> (Value, Value) { |
| assert_eq!(a.layout().ty, b.layout().ty); |
| |
| // c + carry -> c + first intermediate carry or borrow respectively |
| let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b); |
| let c = int0.value_field(fx, FieldIdx::ZERO); |
| let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx); |
| |
| // c + carry -> c + second intermediate carry or borrow respectively |
| let clif_ty = fx.clif_type(a.layout().ty).unwrap(); |
| let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in); |
| let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty)); |
| let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int); |
| let (c, cb1) = int1.load_scalar_pair(fx); |
| |
| // carry0 | carry1 -> carry or borrow respectively |
| let cb_out = fx.bcx.ins().bor(cb0, cb1); |
| |
| (cb_out, c) |
| } |
| |
| enum PackSize { |
| U8, |
| U16, |
| S8, |
| S16, |
| } |
| |
| impl PackSize { |
| fn ret_clif_type(&self) -> Type { |
| match self { |
| Self::U8 | Self::S8 => types::I8, |
| Self::U16 | Self::S16 => types::I16, |
| } |
| } |
| fn src_clif_type(&self) -> Type { |
| match self { |
| Self::U8 | Self::S8 => types::I16, |
| Self::U16 | Self::S16 => types::I32, |
| } |
| } |
| fn src_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { |
| match self { |
| Self::U8 | Self::S8 => tcx.types.i16, |
| Self::U16 | Self::S16 => tcx.types.i32, |
| } |
| } |
| fn ret_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { |
| match self { |
| Self::U8 => tcx.types.u8, |
| Self::S8 => tcx.types.i8, |
| Self::U16 => tcx.types.u16, |
| Self::S16 => tcx.types.i16, |
| } |
| } |
| fn max(&self) -> i64 { |
| match self { |
| Self::U8 => u8::MAX as u64 as i64, |
| Self::S8 => i8::MAX as u8 as u64 as i64, |
| Self::U16 => u16::MAX as u64 as i64, |
| Self::S16 => i16::MAX as u64 as u64 as i64, |
| } |
| } |
| fn min(&self) -> i64 { |
| match self { |
| Self::U8 | Self::U16 => 0, |
| Self::S8 => i16::from(i8::MIN) as u16 as i64, |
| Self::S16 => i32::from(i16::MIN) as u32 as i64, |
| } |
| } |
| } |
| |
| enum PackWidth { |
| Sse = 1, |
| Avx = 2, |
| } |
| impl PackWidth { |
| fn divisor(&self) -> u64 { |
| match self { |
| Self::Sse => 1, |
| Self::Avx => 2, |
| } |
| } |
| } |
| |
| /// Implement an x86 pack instruction with the intrinsic `_mm{,256}pack{us,s}_epi{16,32}`. |
| /// Validated for correctness against LLVM, see commit `c8f5d35508e062bd2d95e6c03429bfec831db6d3`. |
| fn pack_instruction<'tcx>( |
| fx: &mut FunctionCx<'_, '_, 'tcx>, |
| a: CValue<'tcx>, |
| b: CValue<'tcx>, |
| ret: CPlace<'tcx>, |
| ret_size: PackSize, |
| width: PackWidth, |
| ) { |
| assert_eq!(a.layout(), b.layout()); |
| let layout = a.layout(); |
| |
| let (src_lane_count, src_lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
| let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); |
| assert_eq!(src_lane_ty, ret_size.src_ty(fx.tcx)); |
| assert_eq!(ret_lane_ty, ret_size.ret_ty(fx.tcx)); |
| assert_eq!(src_lane_count * 2, ret_lane_count); |
| |
| let min = fx.bcx.ins().iconst(ret_size.src_clif_type(), ret_size.min()); |
| let max = fx.bcx.ins().iconst(ret_size.src_clif_type(), ret_size.max()); |
| let ret_lane_layout = fx.layout_of(ret_size.ret_ty(fx.tcx)); |
| |
| let mut round = |source: CValue<'tcx>, source_offset: u64, dest_offset: u64| { |
| let step_amount = src_lane_count / width.divisor(); |
| let dest_offset = step_amount * dest_offset; |
| for idx in 0..step_amount { |
| let lane = source.value_lane(fx, step_amount * source_offset + idx).load_scalar(fx); |
| let sat = fx.bcx.ins().smax(lane, min); |
| let sat = match ret_size { |
| PackSize::U8 | PackSize::U16 => fx.bcx.ins().umin(sat, max), |
| PackSize::S8 | PackSize::S16 => fx.bcx.ins().smin(sat, max), |
| }; |
| let res = fx.bcx.ins().ireduce(ret_size.ret_clif_type(), sat); |
| let res_lane = CValue::by_val(res, ret_lane_layout); |
| ret.place_lane(fx, dest_offset + idx).write_cvalue(fx, res_lane); |
| } |
| }; |
| |
| round(a, 0, 0); |
| round(b, 0, 1); |
| |
| if let PackWidth::Avx = width { |
| round(a, 1, 2); |
| round(b, 1, 3); |
| } |
| } |
