device-libs: Guard trig reduction quadrant index against NaN UB

adelejjeh · claude · adelejjeh · commit cf5e0cbefca9 · 2026-06-01T17:11:54.000-05:00
Guard the `(int)fn & 0x3` quadrant index computation in trig reduction functions against NaN input. `fptosi NaN` is UB in C and produces `poison` in LLVM IR, which the compiler exploits during constant-folding to return garbage from `cos(inf)`, `sin(inf)`, etc. Fix by adding an isnan check: `isnan(fn) ? 0 : ((int)fn & 0x3)`. The AMDGPU backend folds away the guard at codegen since v_cvt_i32_f32 already returns 0 for NaN (see llvm#200960). Fixes: LCOMPILER-2150 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
diff --git a/amd/device-libs/ocml/src/trigpiredD.cl b/amd/device-libs/ocml/src/trigpiredD.cl
@@ -17,7 +17,7 @@ MATH_PRIVATE(trigpired)(double x)
 
     struct redret ret;
     ret.hi = MATH_MAD(t, -0.5, x);
-    ret.i = (int)t & 0x3;
+    ret.i = BUILTIN_ISNAN_F64(t) ? 0 : ((int)t & 0x3);
     return ret;
 }
 
diff --git a/amd/device-libs/ocml/src/trigpiredF.cl b/amd/device-libs/ocml/src/trigpiredF.cl
@@ -17,7 +17,7 @@ MATH_PRIVATE(trigpired)(float x)
 
     struct redret ret;
     ret.hi = MATH_MAD(t, -0.5f, x);
-    ret.i = (int)t & 0x3;
+    ret.i = BUILTIN_ISNAN_F32(t) ? 0 : ((int)t & 0x3);
     return ret;
 }
 
diff --git a/amd/device-libs/ocml/src/trigpiredH.cl b/amd/device-libs/ocml/src/trigpiredH.cl
@@ -17,7 +17,7 @@ MATH_PRIVATE(trigpired)(half x)
 
     struct redret ret;
     ret.hi = MATH_MAD(t, -0.5h, x);
-    ret.i = (short)t & (short)0x3;
+    ret.i = BUILTIN_ISNAN_F16(t) ? (short)0 : ((short)t & (short)0x3);
     return ret;
 }
 
diff --git a/amd/device-libs/ocml/src/trigredH.cl b/amd/device-libs/ocml/src/trigredH.cl
@@ -21,7 +21,7 @@ MATH_PRIVATE(trigred)(half hx)
 
     struct redret ret;
     ret.hi = (half)BUILTIN_MAD_F32(fn, -pb2_c, BUILTIN_MAD_F32(fn, -pb2_b, BUILTIN_MAD_F32(fn, -pb2_a, x)));
-    ret.i =  (int)fn & 0x3;
+    ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);
     return ret;
 }
 
diff --git a/amd/device-libs/ocml/src/trigredsmallD.cl b/amd/device-libs/ocml/src/trigredsmallD.cl
@@ -30,7 +30,7 @@ MATH_PRIVATE(trigredsmall)(double x)
     struct redret ret;
     ret.hi = rh;
     ret.lo = rt;
-    ret.i = (int)dn & 0x3;
+    ret.i = BUILTIN_ISNAN_F64(dn) ? 0 : ((int)dn & 0x3);
     return ret;
 }
 
diff --git a/amd/device-libs/ocml/src/trigredsmallF.cl b/amd/device-libs/ocml/src/trigredsmallF.cl
@@ -53,7 +53,7 @@ mad_reduce(float x)
 
     struct redret ret;
     ret.hi = MATH_MAD(-piby2_l, fn, r);
-    ret.i = (int)fn & 0x3;
+    ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);
     return ret;
 #endif
 }
@@ -87,7 +87,7 @@ fma_reduce(float x)
     ret.hi = r;
 #endif
 
-    ret.i =(int)fn & 0x3;
+    ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);
     return ret;
 }
 

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ MATH_PRIVATE(trigpired)(double x)`
`17`	`17`
`18`	`18`	`struct redret ret;`
`19`	`19`	`ret.hi = MATH_MAD(t, -0.5, x);`
`20`		`- ret.i = (int)t & 0x3;`
	`20`	`+ ret.i = BUILTIN_ISNAN_F64(t) ? 0 : ((int)t & 0x3);`
`21`	`21`	`return ret;`
`22`	`22`	`}`
`23`	`23`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ MATH_PRIVATE(trigred)(half hx)`
`21`	`21`
`22`	`22`	`struct redret ret;`
`23`	`23`	`ret.hi = (half)BUILTIN_MAD_F32(fn, -pb2_c, BUILTIN_MAD_F32(fn, -pb2_b, BUILTIN_MAD_F32(fn, -pb2_a, x)));`
`24`		`- ret.i = (int)fn & 0x3;`
	`24`	`+ ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);`
`25`	`25`	`return ret;`
`26`	`26`	`}`
`27`	`27`
Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ MATH_PRIVATE(trigredsmall)(double x)`
`30`	`30`	`struct redret ret;`
`31`	`31`	`ret.hi = rh;`
`32`	`32`	`ret.lo = rt;`
`33`		`- ret.i = (int)dn & 0x3;`
	`33`	`+ ret.i = BUILTIN_ISNAN_F64(dn) ? 0 : ((int)dn & 0x3);`
`34`	`34`	`return ret;`
`35`	`35`	`}`
`36`	`36`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ mad_reduce(float x)`
`53`	`53`
`54`	`54`	`struct redret ret;`
`55`	`55`	`ret.hi = MATH_MAD(-piby2_l, fn, r);`
`56`		`- ret.i = (int)fn & 0x3;`
	`56`	`+ ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);`
`57`	`57`	`return ret;`
`58`	`58`	`#endif`
`59`	`59`	`}`
`@@ -87,7 +87,7 @@ fma_reduce(float x)`
`87`	`87`	`ret.hi = r;`
`88`	`88`	`#endif`
`89`	`89`
`90`		`- ret.i =(int)fn & 0x3;`
	`90`	`+ ret.i = BUILTIN_ISNAN_F32(fn) ? 0 : ((int)fn & 0x3);`
`91`	`91`	`return ret;`
`92`	`92`	`}`
`93`	`93`