Skip to content

Commit

Permalink
Don't set fast(-math) for certain simd ops
Browse files Browse the repository at this point in the history 
`fast-math` implies things like functions not being able to accept as an
argument or return as a result, say, `inf` which made these functions
confusingly named or behaving incorrectly, depending on how you
interpret it. Since the time when these intrinsics have been implemented
the intrinsics user's (stdsimd) approach has changed significantly and
so now it is required that these intrinsics operate normally rather than
in "whatever" way.

Fixes #84268
  • Loading branch information
@nagisa
nagisa committed Apr 17, 2021
1 parent cd9b305 commit 487e273
Show file tree
Hide file tree
Showing 19 changed files with 154 additions and 131 deletions.
18 changes: 9 additions & 9 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,39 +261,39 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1242,14 +1242,14 @@ impl Builder<'a, 'll, 'tcx> {
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1282,15 +1282,15 @@ impl Builder<'a, 'll, 'tcx> {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down
40 changes: 19 additions & 21 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1053,50 +1053,48 @@ fn generic_simd_intrinsic(
let vec_ty = bx.type_vector(elem_ty, in_len);

let (intr_name, fn_ty) = match name {
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
_ => return_error!("unrecognized intrinsic `{}`", name),
};

let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
Ok(c)
}

if std::matches!(
name,
sym::simd_fsqrt
| sym::simd_fsin
| sym::simd_fcos
sym::simd_ceil
| sym::simd_fabs
| sym::simd_ceil
| sym::simd_floor
| sym::simd_round
| sym::simd_trunc
| sym::simd_fexp
| sym::simd_fcos
| sym::simd_fexp2
| sym::simd_fexp
| sym::simd_flog10
| sym::simd_flog2
| sym::simd_flog
| sym::simd_fpowi
| sym::simd_fpow
| sym::simd_floor
| sym::simd_fma
| sym::simd_fpow
| sym::simd_fpowi
| sym::simd_fsin
| sym::simd_fsqrt
| sym::simd_round
| sym::simd_trunc
) {
return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
}
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1354,7 +1354,7 @@ extern "C" {
pub fn LLVMBuildNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildFNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildNot(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMRustSetHasUnsafeAlgebra(Instr: &Value);
pub fn LLVMRustSetFastMath(Instr: &Value);

// Memory
pub fn LLVMBuildAlloca(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
Expand Down
6 changes: 4 additions & 2 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,8 +349,10 @@ extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn,
F->setAttributes(PALNew);
}

// enable fpmath flag UnsafeAlgebra
extern "C" void LLVMRustSetHasUnsafeAlgebra(LLVMValueRef V) {
// Enable a fast-math flag
//
// https://llvm.org/docs/LangRef.html#fast-math-flags
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
I->setFast(true);
}
Expand Down
23 changes: 23 additions & 0 deletions src/test/codegen/issue-84268.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// compile-flags: -O --crate-type=rlib
#![feature(platform_intrinsics, repr_simd)]

extern "platform-intrinsic" {
fn simd_fabs<T>(x: T) -> T;
fn simd_eq<T, U>(x: T, y: T) -> U;
}

#[repr(simd)]
pub struct V([f32; 4]);

#[repr(simd)]
pub struct M([i32; 4]);

#[no_mangle]
// CHECK-LABEL: @is_infinite
pub fn is_infinite(v: V) -> M {
// CHECK: fabs
// CHECK: cmp oeq
unsafe {
simd_eq(simd_fabs(v), V([f32::INFINITY; 4]))
}
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-abs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fabs_32x2
#[no_mangle]
pub unsafe fn fabs_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.fabs.v2f32
// CHECK: call <2 x float> @llvm.fabs.v2f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x4
#[no_mangle]
pub unsafe fn fabs_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.fabs.v4f32
// CHECK: call <4 x float> @llvm.fabs.v4f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x8
#[no_mangle]
pub unsafe fn fabs_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.fabs.v8f32
// CHECK: call <8 x float> @llvm.fabs.v8f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x16
#[no_mangle]
pub unsafe fn fabs_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.fabs.v16f32
// CHECK: call <16 x float> @llvm.fabs.v16f32
simd_fabs(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fabs_64x4
#[no_mangle]
pub unsafe fn fabs_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.fabs.v4f64
// CHECK: call <4 x double> @llvm.fabs.v4f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x2
#[no_mangle]
pub unsafe fn fabs_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.fabs.v2f64
// CHECK: call <2 x double> @llvm.fabs.v2f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x8
#[no_mangle]
pub unsafe fn fabs_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.fabs.v8f64
// CHECK: call <8 x double> @llvm.fabs.v8f64
simd_fabs(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-ceil.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @ceil_32x2
#[no_mangle]
pub unsafe fn ceil_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.ceil.v2f32
// CHECK: call <2 x float> @llvm.ceil.v2f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x4
#[no_mangle]
pub unsafe fn ceil_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.ceil.v4f32
// CHECK: call <4 x float> @llvm.ceil.v4f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x8
#[no_mangle]
pub unsafe fn ceil_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.ceil.v8f32
// CHECK: call <8 x float> @llvm.ceil.v8f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x16
#[no_mangle]
pub unsafe fn ceil_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.ceil.v16f32
// CHECK: call <16 x float> @llvm.ceil.v16f32
simd_ceil(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @ceil_64x4
#[no_mangle]
pub unsafe fn ceil_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.ceil.v4f64
// CHECK: call <4 x double> @llvm.ceil.v4f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x2
#[no_mangle]
pub unsafe fn ceil_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.ceil.v2f64
// CHECK: call <2 x double> @llvm.ceil.v2f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x8
#[no_mangle]
pub unsafe fn ceil_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.ceil.v8f64
// CHECK: call <8 x double> @llvm.ceil.v8f64
simd_ceil(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-cos.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fcos_32x2
#[no_mangle]
pub unsafe fn fcos_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.cos.v2f32
// CHECK: call <2 x float> @llvm.cos.v2f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x4
#[no_mangle]
pub unsafe fn fcos_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.cos.v4f32
// CHECK: call <4 x float> @llvm.cos.v4f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x8
#[no_mangle]
pub unsafe fn fcos_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.cos.v8f32
// CHECK: call <8 x float> @llvm.cos.v8f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x16
#[no_mangle]
pub unsafe fn fcos_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.cos.v16f32
// CHECK: call <16 x float> @llvm.cos.v16f32
simd_fcos(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fcos_64x4
#[no_mangle]
pub unsafe fn fcos_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.cos.v4f64
// CHECK: call <4 x double> @llvm.cos.v4f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x2
#[no_mangle]
pub unsafe fn fcos_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.cos.v2f64
// CHECK: call <2 x double> @llvm.cos.v2f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x8
#[no_mangle]
pub unsafe fn fcos_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.cos.v8f64
// CHECK: call <8 x double> @llvm.cos.v8f64
simd_fcos(a)
}
Loading

0 comments on commit 487e273

Please sign in to comment.