diff --git a/sse2neon.h b/sse2neon.h index 16ef2f83..4cf360d8 100644 --- a/sse2neon.h +++ b/sse2neon.h @@ -755,7 +755,9 @@ FORCE_INLINE void _sse2neon_smp_mb(void) #endif /* Flush-to-zero (FTZ) mode macros. * On x86, FTZ (MXCSR bit 15) flushes denormal outputs to zero. - * On ARM, FPCR/FPSCR bit 24 provides unified FZ+DAZ behavior. + * On ARM without FEAT_AFP: FPCR.FZ (bit 24) flushes both inputs and outputs. + * On ARM with FEAT_AFP (ARMv8.7+): FPCR.FZ flushes only outputs when AH=1, + * allowing independent control of input (FIZ) and output (FZ) flushing. * ARMv7 NEON: Per ARM ARM, Advanced SIMD has "Flush-to-zero mode always * enabled" - denormals flush regardless of FPSCR.FZ (some impls may vary). * ARMv8: FPCR.FZ correctly controls denormal handling for NEON ops. @@ -771,8 +773,10 @@ FORCE_INLINE void _sse2neon_smp_mb(void) #endif /* Denormals-are-zero (DAZ) mode macros. * On x86, DAZ (MXCSR bit 6) treats denormal inputs as zero. - * On ARM, setting DAZ enables the same FPCR/FPSCR bit 24 as FTZ, - * providing unified handling for both input and output denormals. + * On ARM without FEAT_AFP: setting DAZ enables FPCR.FZ (bit 24), which also + * affects outputs - the best approximation available pre-ARMv8.7. + * On ARM with FEAT_AFP (ARMv8.7+): DAZ is mapped to FPCR.FIZ (bit 0) with + * FPCR.AH (bit 1) set, giving true input-only denormal flushing. */ #ifndef _MM_DENORMALS_ZERO_MASK #define _MM_DENORMALS_ZERO_MASK 0x0040 @@ -1456,7 +1460,14 @@ enum _mm_hint { // The bit field mapping to the FPCR(floating-point control register) typedef struct { +#if defined(__ARM_FEATURE_AFP) + // FEAT_AFP (ARMv8.7+): independent input/output denormal flush control + uint16_t bit0_fiz : 1; // bit 0: FIZ - Flush Inputs to Zero + uint16_t bit1_ah : 1; // bit 1: AH - Alternate Handling + uint16_t res0 : 14; // bits 2-15 +#else uint16_t res0; +#endif uint8_t res1 : 6; uint8_t bit22 : 1; uint8_t bit23 : 1; @@ -3310,6 +3321,11 @@ FORCE_INLINE void _sse2neon_mm_set_flush_zero_mode(unsigned int flag) #endif r.field.bit24 = (flag & _MM_FLUSH_ZERO_MASK) == _MM_FLUSH_ZERO_ON; +#if defined(__ARM_FEATURE_AFP) + // With FEAT_AFP, AH=1 makes FZ flush outputs only (correct FTZ semantics). + // Keep AH set if either FZ or FIZ is active. + r.field.bit1_ah = r.field.bit24 | r.field.bit0_fiz; +#endif #if SSE2NEON_ARCH_AARCH64 _sse2neon_set_fpcr(r.value); @@ -3387,7 +3403,8 @@ FORCE_INLINE __m128 _mm_set1_ps(float _w) // Supported MXCSR fields: // - Bits 13-14: Rounding mode (RM) - SUPPORTED via ARM FPCR/FPSCR // - Bit 15 (FZ): Flush-to-zero mode - SUPPORTED via ARM FPCR/FPSCR bit 24 -// - Bit 6 (DAZ): Denormals-are-zero mode - SUPPORTED (unified with FZ on ARM) +// - Bit 6 (DAZ): Denormals-are-zero mode - SUPPORTED independently via +// FPCR.FIZ+AH on FEAT_AFP (ARMv8.7+), or unified with FZ on older ARM. // // Unsupported MXCSR fields (silently ignored): // - Bits 0-5: Exception flags (IE, DE, ZE, OE, UE, PE) - NOT EMULATED @@ -3395,17 +3412,32 @@ FORCE_INLINE __m128 _mm_set1_ps(float _w) // See "MXCSR Exception Flags - NOT EMULATED" documentation block for details. // // ARM Platform Behavior: -// - ARM FPCR/FPSCR bit 24 provides unified FZ+DAZ behavior. Setting either -// _MM_FLUSH_ZERO_ON or _MM_DENORMALS_ZERO_ON enables the same ARM bit. +// - Without FEAT_AFP: FPCR.FZ is set if either FTZ or DAZ is requested +// (unified, slightly over-approximates when only one flag is set). +// - With FEAT_AFP (ARMv8.7+): FTZ maps to FPCR.FZ and DAZ maps to FPCR.FIZ, +// both requiring FPCR.AH=1 for correct output-only / input-only semantics. // - ARMv7 NEON: "Flush-to-zero mode always enabled" per ARM ARM (impl may vary) -// - ARMv8: FPCR.FZ correctly controls denormal handling for NEON operations FORCE_INLINE void _mm_setcsr(unsigned int a) { _MM_SET_ROUNDING_MODE(a & _MM_ROUND_MASK); - // ARM FPCR.bit24 handles both FZ and DAZ - set if either is requested +#if defined(__ARM_FEATURE_AFP) + // FEAT_AFP: set FZ and FIZ independently; AH=1 enables output-only FZ and + // input-only FIZ semantics. + union { + fpcr_bitfield field; + uint64_t value; + } r; + r.value = _sse2neon_get_fpcr(); + r.field.bit24 = (a & _MM_FLUSH_ZERO_MASK) == _MM_FLUSH_ZERO_ON; + r.field.bit0_fiz = (a & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON; + r.field.bit1_ah = r.field.bit24 | r.field.bit0_fiz; + _sse2neon_set_fpcr(r.value); +#else + // Without FEAT_AFP: FPCR.FZ handles both; set if either flag is requested. _MM_SET_FLUSH_ZERO_MODE( (a & _MM_FLUSH_ZERO_MASK) | ((a & _MM_DENORMALS_ZERO_MASK) ? _MM_FLUSH_ZERO_ON : 0)); +#endif } // Get the unsigned 32-bit value of the MXCSR control and status register. @@ -3413,8 +3445,9 @@ FORCE_INLINE void _mm_setcsr(unsigned int a) // // Returned MXCSR fields: // - Bits 13-14: Rounding mode (RM) - Reflects current ARM FPCR/FPSCR setting -// - Bit 15 (FZ): Flush-to-zero mode - Reflects ARM FPCR/FPSCR bit 24 -// - Bit 6 (DAZ): Denormals-are-zero mode - Mirrors FZ (unified on ARM) +// - Bit 15 (FZ): Flush-to-zero mode - Reflects FPCR.FZ (bit 24) +// - Bit 6 (DAZ): Denormals-are-zero mode - Reflects FPCR.FIZ (FEAT_AFP) or +// mirrors FZ (pre-ARMv8.7 fallback) // // Fields always returned as zero (NOT EMULATED): // - Bits 0-5: Exception flags - ALWAYS 0 (exceptions not tracked) @@ -3423,13 +3456,25 @@ FORCE_INLINE void _mm_setcsr(unsigned int a) // details. // // ARM Platform Behavior: -// - When ARM FPCR/FPSCR bit 24 is enabled, both FZ and DAZ bits are reported -// as set (the original setting cannot be distinguished). +// - With FEAT_AFP: FZ and DAZ are read back independently from FPCR.FZ and +// FPCR.FIZ, faithfully reflecting what was set. +// - Without FEAT_AFP: When FPCR.FZ is enabled, both FZ and DAZ bits are +// reported as set (the original setting cannot be distinguished). // - ARMv7 NEON: Returned bits reflect FPSCR, but NEON always flushes denormals FORCE_INLINE unsigned int _mm_getcsr(void) { +#if defined(__ARM_FEATURE_AFP) + union { + fpcr_bitfield field; + uint64_t value; + } r; + r.value = _sse2neon_get_fpcr(); + return _MM_GET_ROUNDING_MODE() | (r.field.bit24 ? _MM_FLUSH_ZERO_ON : 0) | + (r.field.bit0_fiz ? _MM_DENORMALS_ZERO_ON : 0); +#else return _MM_GET_ROUNDING_MODE() | _MM_GET_FLUSH_ZERO_MODE() | _MM_GET_DENORMALS_ZERO_MODE(); +#endif } // Set packed single-precision (32-bit) floating-point elements in dst with the @@ -11602,7 +11647,12 @@ FORCE_INLINE unsigned int _sse2neon_mm_get_denormals_zero_mode(void) __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ #endif +#if defined(__ARM_FEATURE_AFP) + // With FEAT_AFP, DAZ is represented by FIZ (input flush), not FZ. + return r.field.bit0_fiz ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF; +#else return r.field.bit24 ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF; +#endif } // Count the number of bits set to 1 in unsigned 32-bit integer a, and @@ -11683,7 +11733,15 @@ FORCE_INLINE void _sse2neon_mm_set_denormals_zero_mode(unsigned int flag) __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ #endif +#if defined(__ARM_FEATURE_AFP) + // With FEAT_AFP, DAZ maps to FIZ (input-only flush). AH=1 is required for + // both FZ and FIZ to operate in their independent output/input modes. + r.field.bit0_fiz = + (flag & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON; + r.field.bit1_ah = r.field.bit0_fiz | r.field.bit24; +#else r.field.bit24 = (flag & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON; +#endif #if SSE2NEON_ARCH_AARCH64 _sse2neon_set_fpcr(r.value); diff --git a/tests/ieee754.cpp b/tests/ieee754.cpp index d61fdfcb..5153ac43 100644 --- a/tests/ieee754.cpp +++ b/tests/ieee754.cpp @@ -1024,8 +1024,20 @@ TEST_CASE(ftz_getcsr_roundtrip) printf(" FAILED: FTZ ON - FZ bit not set (line %d)\n", __LINE__); ret = TEST_FAIL; } -#if defined(__aarch64__) || defined(__arm__) - /* On ARM, FZ and DAZ share bit 24, so enabling FZ also enables DAZ */ +#if defined(__ARM_FEATURE_AFP) + /* With FEAT_AFP, FZ and DAZ are independent; setting FTZ should NOT set + * DAZ */ + if (ret == TEST_SUCCESS && + (csr & _MM_DENORMALS_ZERO_MASK) != _MM_DENORMALS_ZERO_OFF) { + printf( + " FAILED: FTZ ON - DAZ should be independent on AFP (line " + "%d)\n", + __LINE__); + ret = TEST_FAIL; + } +#elif defined(__aarch64__) || defined(__arm__) + /* Without FEAT_AFP, ARM FZ and DAZ share bit 24, so enabling FZ also + * enables DAZ */ if (ret == TEST_SUCCESS && (csr & _MM_DENORMALS_ZERO_MASK) != _MM_DENORMALS_ZERO_ON) { printf(" FAILED: FTZ ON - DAZ not coupled on ARM (line %d)\n", @@ -1054,8 +1066,20 @@ TEST_CASE(ftz_getcsr_roundtrip) __LINE__); ret = TEST_FAIL; } -#if defined(__aarch64__) || defined(__arm__) - /* On ARM, DAZ and FZ share bit 24, so enabling DAZ also enables FZ */ +#if defined(__ARM_FEATURE_AFP) + /* With FEAT_AFP, FZ and DAZ are independent; setting DAZ should NOT + * set FTZ */ + if (ret == TEST_SUCCESS && + (csr & _MM_FLUSH_ZERO_MASK) != _MM_FLUSH_ZERO_OFF) { + printf( + " FAILED: DAZ ON - FZ should be independent on AFP (line " + "%d)\n", + __LINE__); + ret = TEST_FAIL; + } +#elif defined(__aarch64__) || defined(__arm__) + /* Without FEAT_AFP, ARM DAZ and FZ share bit 24, so enabling DAZ also + * enables FZ */ if (ret == TEST_SUCCESS && (csr & _MM_FLUSH_ZERO_MASK) != _MM_FLUSH_ZERO_ON) { printf(" FAILED: DAZ ON - FZ not coupled on ARM (line %d)\n", @@ -1096,6 +1120,104 @@ TEST_CASE(ftz_getcsr_roundtrip) return ret; } +#if defined(__ARM_FEATURE_AFP) +/* With FEAT_AFP, FTZ (output flush) and DAZ (input flush) are independent. + * These tests verify the correct separation that pre-ARMv8.7 ARM could not + * provide. They are compiled only when __ARM_FEATURE_AFP is defined, which + * requires -march=armv8.7-a (or newer) at compile time. */ + +TEST_CASE(afp_ftz_only_no_input_flush) +{ + /* FTZ-only: denormal *inputs* must NOT be flushed to zero. + * Operation: denorm * 2.0 should give a non-zero result (the denormal is + * used as-is), whereas DAZ would treat the input as 0 giving 0 * 2 = 0. + */ + unsigned int original_csr = _mm_getcsr(); + result_t ret = TEST_SUCCESS; + + float denorm = make_denormal(); + float factor = 2.0f; + __m128 a = _mm_set1_ps(denorm); + __m128 b = _mm_set1_ps(factor); + + /* Enable FTZ only, DAZ off */ + _mm_setcsr( + (original_csr & ~static_cast(_MM_FLUSH_ZERO_MASK | + _MM_DENORMALS_ZERO_MASK)) | + _MM_FLUSH_ZERO_ON); + + __m128 c = _mm_mul_ps(a, b); + float result = extract_ps(c, 0); + + /* denorm * 2 = a slightly larger denormal - still non-zero (FTZ only + * flushes outputs, and denorm*2 is still denormal so it IS flushed to 0). + * Actually denorm*2 IS a denormal output, so FTZ will flush it to 0. + * The real test: the *input* denorm was not silently zeroed before the + * multiply. We verify this by checking that the computation ran (FTZ + * flushes the output to 0, not because DAZ zeroed the input). + * Use denorm * 1.0: the output equals the input, which FTZ flushes to 0, + * but the path taken is different from DAZ (input zeroed -> 0 * 1 = 0). + * To distinguish FTZ-only from DAZ: use make_half_flt_min() * 2 = FLT_MIN + * (normal output), which should NOT be flushed by FTZ. */ + float denorm2 = make_half_flt_min(); /* 2^-127, a denormal */ + __m128 a2 = _mm_set1_ps(denorm2); + __m128 b2 = _mm_set1_ps(2.0f); + __m128 c2 = _mm_mul_ps(a2, b2); + float result2 = extract_ps(c2, 0); + + /* denorm2 * 2 = FLT_MIN (a normal number), so FTZ must NOT flush it. + * If DAZ had zeroed the input we would get 0 * 2 = 0 instead. */ + if (result2 != FLT_MIN) { + printf( + " FAILED: FTZ-only input not preserved: got %e, want %e " + "(line %d)\n", + (double) result2, (double) FLT_MIN, __LINE__); + ret = TEST_FAIL; + } + (void) result; + + _mm_setcsr(original_csr); + return ret; +} + +TEST_CASE(afp_daz_only_no_output_flush) +{ + /* DAZ-only: denormal *outputs* must NOT be flushed to zero. + * Operation: FLT_MIN * 0.5 produces a denormal output. With FTZ that would + * be flushed to 0. With DAZ-only it must remain as the denormal value. + */ + unsigned int original_csr = _mm_getcsr(); + result_t ret = TEST_SUCCESS; + + float min_normal = FLT_MIN; + float half = 0.5f; + __m128 a = _mm_set1_ps(min_normal); + __m128 b = _mm_set1_ps(half); + + /* Enable DAZ only, FTZ off */ + _mm_setcsr( + (original_csr & ~static_cast(_MM_FLUSH_ZERO_MASK | + _MM_DENORMALS_ZERO_MASK)) | + _MM_DENORMALS_ZERO_ON); + + __m128 c = _mm_mul_ps(a, b); + float result = extract_ps(c, 0); + + /* FLT_MIN * 0.5 = FLT_MIN/2 (a denormal). The input FLT_MIN is normal so + * DAZ does not affect it. Without FTZ the denormal output must survive. */ + float expected = make_half_flt_min(); + if (result != expected) { + printf( + " FAILED: DAZ-only output flushed: got %e, want %e (line %d)\n", + (double) result, (double) expected, __LINE__); + ret = TEST_FAIL; + } + + _mm_setcsr(original_csr); + return ret; +} +#endif /* __ARM_FEATURE_AFP */ + /* Comparison Tests with Special Values */ TEST_CASE(cmpord_ps_nan) @@ -1885,6 +2007,10 @@ int main(void) RUN_TEST(daz_input_flush); RUN_TEST(ftz_daz_disabled); RUN_TEST(ftz_getcsr_roundtrip); +#if defined(__ARM_FEATURE_AFP) + RUN_TEST(afp_ftz_only_no_input_flush); + RUN_TEST(afp_daz_only_no_output_flush); +#endif printf("\n--- Comparison Tests ---\n"); RUN_TEST(cmpord_ps_nan);