Skip to content

arm: support FEAT_AFP FTZ/DAZ semantics and tests #766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
howjmay wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

arm: support FEAT_AFP FTZ/DAZ semantics and tests #766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 70 additions & 12 deletions sse2neon.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,7 +755,9 @@ FORCE_INLINE void _sse2neon_smp_mb(void)
#endif
/* Flush-to-zero (FTZ) mode macros.
* On x86, FTZ (MXCSR bit 15) flushes denormal outputs to zero.
* On ARM, FPCR/FPSCR bit 24 provides unified FZ+DAZ behavior.
* On ARM without FEAT_AFP: FPCR.FZ (bit 24) flushes both inputs and outputs.
* On ARM with FEAT_AFP (ARMv8.7+): FPCR.FZ flushes only outputs when AH=1,
* allowing independent control of input (FIZ) and output (FZ) flushing.
* ARMv7 NEON: Per ARM ARM, Advanced SIMD has "Flush-to-zero mode always
* enabled" - denormals flush regardless of FPSCR.FZ (some impls may vary).
* ARMv8: FPCR.FZ correctly controls denormal handling for NEON ops.
Expand All @@ -771,8 +773,10 @@ FORCE_INLINE void _sse2neon_smp_mb(void)
#endif
/* Denormals-are-zero (DAZ) mode macros.
* On x86, DAZ (MXCSR bit 6) treats denormal inputs as zero.
* On ARM, setting DAZ enables the same FPCR/FPSCR bit 24 as FTZ,
* providing unified handling for both input and output denormals.
* On ARM without FEAT_AFP: setting DAZ enables FPCR.FZ (bit 24), which also
* affects outputs - the best approximation available pre-ARMv8.7.
* On ARM with FEAT_AFP (ARMv8.7+): DAZ is mapped to FPCR.FIZ (bit 0) with
* FPCR.AH (bit 1) set, giving true input-only denormal flushing.
*/
#ifndef _MM_DENORMALS_ZERO_MASK
#define _MM_DENORMALS_ZERO_MASK 0x0040
Expand Down Expand Up @@ -1456,7 +1460,14 @@ enum _mm_hint {

// The bit field mapping to the FPCR(floating-point control register)
typedef struct {
#if defined(__ARM_FEATURE_AFP)
// FEAT_AFP (ARMv8.7+): independent input/output denormal flush control
uint16_t bit0_fiz : 1; // bit 0: FIZ - Flush Inputs to Zero
uint16_t bit1_ah : 1; // bit 1: AH - Alternate Handling
uint16_t res0 : 14; // bits 2-15
#else
uint16_t res0;
#endif
uint8_t res1 : 6;
uint8_t bit22 : 1;
uint8_t bit23 : 1;
Expand Down Expand Up @@ -3310,6 +3321,11 @@ FORCE_INLINE void _sse2neon_mm_set_flush_zero_mode(unsigned int flag)
#endif

r.field.bit24 = (flag & _MM_FLUSH_ZERO_MASK) == _MM_FLUSH_ZERO_ON;
#if defined(__ARM_FEATURE_AFP)
// With FEAT_AFP, AH=1 makes FZ flush outputs only (correct FTZ semantics).
// Keep AH set if either FZ or FIZ is active.
r.field.bit1_ah = r.field.bit24 | r.field.bit0_fiz;
#endif

#if SSE2NEON_ARCH_AARCH64
_sse2neon_set_fpcr(r.value);
Expand Down Expand Up @@ -3387,34 +3403,51 @@ FORCE_INLINE __m128 _mm_set1_ps(float _w)
// Supported MXCSR fields:
// - Bits 13-14: Rounding mode (RM) - SUPPORTED via ARM FPCR/FPSCR
// - Bit 15 (FZ): Flush-to-zero mode - SUPPORTED via ARM FPCR/FPSCR bit 24
// - Bit 6 (DAZ): Denormals-are-zero mode - SUPPORTED (unified with FZ on ARM)
// - Bit 6 (DAZ): Denormals-are-zero mode - SUPPORTED independently via
// FPCR.FIZ+AH on FEAT_AFP (ARMv8.7+), or unified with FZ on older ARM.
//
// Unsupported MXCSR fields (silently ignored):
// - Bits 0-5: Exception flags (IE, DE, ZE, OE, UE, PE) - NOT EMULATED
// - Bits 7-12: Exception masks - NOT EMULATED
// See "MXCSR Exception Flags - NOT EMULATED" documentation block for details.
//
// ARM Platform Behavior:
// - ARM FPCR/FPSCR bit 24 provides unified FZ+DAZ behavior. Setting either
// _MM_FLUSH_ZERO_ON or _MM_DENORMALS_ZERO_ON enables the same ARM bit.
// - Without FEAT_AFP: FPCR.FZ is set if either FTZ or DAZ is requested
// (unified, slightly over-approximates when only one flag is set).
// - With FEAT_AFP (ARMv8.7+): FTZ maps to FPCR.FZ and DAZ maps to FPCR.FIZ,
// both requiring FPCR.AH=1 for correct output-only / input-only semantics.
// - ARMv7 NEON: "Flush-to-zero mode always enabled" per ARM ARM (impl may vary)
// - ARMv8: FPCR.FZ correctly controls denormal handling for NEON operations
FORCE_INLINE void _mm_setcsr(unsigned int a)
{
_MM_SET_ROUNDING_MODE(a & _MM_ROUND_MASK);
// ARM FPCR.bit24 handles both FZ and DAZ - set if either is requested
#if defined(__ARM_FEATURE_AFP)
// FEAT_AFP: set FZ and FIZ independently; AH=1 enables output-only FZ and
// input-only FIZ semantics.
union {
fpcr_bitfield field;
uint64_t value;
} r;
r.value = _sse2neon_get_fpcr();
r.field.bit24 = (a & _MM_FLUSH_ZERO_MASK) == _MM_FLUSH_ZERO_ON;
r.field.bit0_fiz = (a & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON;
r.field.bit1_ah = r.field.bit24 | r.field.bit0_fiz;
_sse2neon_set_fpcr(r.value);
#else
// Without FEAT_AFP: FPCR.FZ handles both; set if either flag is requested.
_MM_SET_FLUSH_ZERO_MODE(
(a & _MM_FLUSH_ZERO_MASK) |
((a & _MM_DENORMALS_ZERO_MASK) ? _MM_FLUSH_ZERO_ON : 0));
#endif
}

// Get the unsigned 32-bit value of the MXCSR control and status register.
// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_getcsr
//
// Returned MXCSR fields:
// - Bits 13-14: Rounding mode (RM) - Reflects current ARM FPCR/FPSCR setting
// - Bit 15 (FZ): Flush-to-zero mode - Reflects ARM FPCR/FPSCR bit 24
// - Bit 6 (DAZ): Denormals-are-zero mode - Mirrors FZ (unified on ARM)
// - Bit 15 (FZ): Flush-to-zero mode - Reflects FPCR.FZ (bit 24)
// - Bit 6 (DAZ): Denormals-are-zero mode - Reflects FPCR.FIZ (FEAT_AFP) or
// mirrors FZ (pre-ARMv8.7 fallback)
//
// Fields always returned as zero (NOT EMULATED):
// - Bits 0-5: Exception flags - ALWAYS 0 (exceptions not tracked)
Expand All @@ -3423,13 +3456,25 @@ FORCE_INLINE void _mm_setcsr(unsigned int a)
// details.
//
// ARM Platform Behavior:
// - When ARM FPCR/FPSCR bit 24 is enabled, both FZ and DAZ bits are reported
// as set (the original setting cannot be distinguished).
// - With FEAT_AFP: FZ and DAZ are read back independently from FPCR.FZ and
// FPCR.FIZ, faithfully reflecting what was set.
// - Without FEAT_AFP: When FPCR.FZ is enabled, both FZ and DAZ bits are
// reported as set (the original setting cannot be distinguished).
// - ARMv7 NEON: Returned bits reflect FPSCR, but NEON always flushes denormals
FORCE_INLINE unsigned int _mm_getcsr(void)
{
#if defined(__ARM_FEATURE_AFP)
union {
fpcr_bitfield field;
uint64_t value;
} r;
r.value = _sse2neon_get_fpcr();
return _MM_GET_ROUNDING_MODE() | (r.field.bit24 ? _MM_FLUSH_ZERO_ON : 0) |
(r.field.bit0_fiz ? _MM_DENORMALS_ZERO_ON : 0);
#else
return _MM_GET_ROUNDING_MODE() | _MM_GET_FLUSH_ZERO_MODE() |
_MM_GET_DENORMALS_ZERO_MODE();
#endif
}

// Set packed single-precision (32-bit) floating-point elements in dst with the
Expand Down Expand Up @@ -11602,7 +11647,12 @@ FORCE_INLINE unsigned int _sse2neon_mm_get_denormals_zero_mode(void)
__asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */
#endif

#if defined(__ARM_FEATURE_AFP)
// With FEAT_AFP, DAZ is represented by FIZ (input flush), not FZ.
return r.field.bit0_fiz ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF;
#else
return r.field.bit24 ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF;
#endif
}

// Count the number of bits set to 1 in unsigned 32-bit integer a, and
Expand Down Expand Up @@ -11683,7 +11733,15 @@ FORCE_INLINE void _sse2neon_mm_set_denormals_zero_mode(unsigned int flag)
__asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */
#endif

#if defined(__ARM_FEATURE_AFP)
// With FEAT_AFP, DAZ maps to FIZ (input-only flush). AH=1 is required for
// both FZ and FIZ to operate in their independent output/input modes.
r.field.bit0_fiz =
(flag & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON;
r.field.bit1_ah = r.field.bit0_fiz | r.field.bit24;
#else
r.field.bit24 = (flag & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON;
#endif

#if SSE2NEON_ARCH_AARCH64
_sse2neon_set_fpcr(r.value);
Expand Down
134 changes: 130 additions & 4 deletions tests/ieee754.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1024,8 +1024,20 @@ TEST_CASE(ftz_getcsr_roundtrip)
printf(" FAILED: FTZ ON - FZ bit not set (line %d)\n", __LINE__);
ret = TEST_FAIL;
}
#if defined(__aarch64__) || defined(__arm__)
/* On ARM, FZ and DAZ share bit 24, so enabling FZ also enables DAZ */
#if defined(__ARM_FEATURE_AFP)
/* With FEAT_AFP, FZ and DAZ are independent; setting FTZ should NOT set
* DAZ */
if (ret == TEST_SUCCESS &&
(csr & _MM_DENORMALS_ZERO_MASK) != _MM_DENORMALS_ZERO_OFF) {
printf(
" FAILED: FTZ ON - DAZ should be independent on AFP (line "
"%d)\n",
__LINE__);
ret = TEST_FAIL;
}
#elif defined(__aarch64__) || defined(__arm__)
/* Without FEAT_AFP, ARM FZ and DAZ share bit 24, so enabling FZ also
* enables DAZ */
if (ret == TEST_SUCCESS &&
(csr & _MM_DENORMALS_ZERO_MASK) != _MM_DENORMALS_ZERO_ON) {
printf(" FAILED: FTZ ON - DAZ not coupled on ARM (line %d)\n",
Expand Down Expand Up @@ -1054,8 +1066,20 @@ TEST_CASE(ftz_getcsr_roundtrip)
__LINE__);
ret = TEST_FAIL;
}
#if defined(__aarch64__) || defined(__arm__)
/* On ARM, DAZ and FZ share bit 24, so enabling DAZ also enables FZ */
#if defined(__ARM_FEATURE_AFP)
/* With FEAT_AFP, FZ and DAZ are independent; setting DAZ should NOT
* set FTZ */
if (ret == TEST_SUCCESS &&
(csr & _MM_FLUSH_ZERO_MASK) != _MM_FLUSH_ZERO_OFF) {
printf(
" FAILED: DAZ ON - FZ should be independent on AFP (line "
"%d)\n",
__LINE__);
ret = TEST_FAIL;
}
#elif defined(__aarch64__) || defined(__arm__)
/* Without FEAT_AFP, ARM DAZ and FZ share bit 24, so enabling DAZ also
* enables FZ */
if (ret == TEST_SUCCESS &&
(csr & _MM_FLUSH_ZERO_MASK) != _MM_FLUSH_ZERO_ON) {
printf(" FAILED: DAZ ON - FZ not coupled on ARM (line %d)\n",
Expand Down Expand Up @@ -1096,6 +1120,104 @@ TEST_CASE(ftz_getcsr_roundtrip)
return ret;
}

#if defined(__ARM_FEATURE_AFP)
/* With FEAT_AFP, FTZ (output flush) and DAZ (input flush) are independent.
* These tests verify the correct separation that pre-ARMv8.7 ARM could not
* provide. They are compiled only when __ARM_FEATURE_AFP is defined, which
* requires -march=armv8.7-a (or newer) at compile time. */

TEST_CASE(afp_ftz_only_no_input_flush)
{
/* FTZ-only: denormal *inputs* must NOT be flushed to zero.
* Operation: denorm * 2.0 should give a non-zero result (the denormal is
* used as-is), whereas DAZ would treat the input as 0 giving 0 * 2 = 0.
*/
unsigned int original_csr = _mm_getcsr();
result_t ret = TEST_SUCCESS;

float denorm = make_denormal();
float factor = 2.0f;
__m128 a = _mm_set1_ps(denorm);
__m128 b = _mm_set1_ps(factor);

/* Enable FTZ only, DAZ off */
_mm_setcsr(
(original_csr & ~static_cast<unsigned>(_MM_FLUSH_ZERO_MASK |
_MM_DENORMALS_ZERO_MASK)) |
_MM_FLUSH_ZERO_ON);

__m128 c = _mm_mul_ps(a, b);
float result = extract_ps(c, 0);

/* denorm * 2 = a slightly larger denormal - still non-zero (FTZ only
* flushes outputs, and denorm*2 is still denormal so it IS flushed to 0).
* Actually denorm*2 IS a denormal output, so FTZ will flush it to 0.
* The real test: the *input* denorm was not silently zeroed before the
* multiply. We verify this by checking that the computation ran (FTZ
* flushes the output to 0, not because DAZ zeroed the input).
* Use denorm * 1.0: the output equals the input, which FTZ flushes to 0,
* but the path taken is different from DAZ (input zeroed -> 0 * 1 = 0).
* To distinguish FTZ-only from DAZ: use make_half_flt_min() * 2 = FLT_MIN
* (normal output), which should NOT be flushed by FTZ. */
float denorm2 = make_half_flt_min(); /* 2^-127, a denormal */
__m128 a2 = _mm_set1_ps(denorm2);
__m128 b2 = _mm_set1_ps(2.0f);
__m128 c2 = _mm_mul_ps(a2, b2);
float result2 = extract_ps(c2, 0);

/* denorm2 * 2 = FLT_MIN (a normal number), so FTZ must NOT flush it.
* If DAZ had zeroed the input we would get 0 * 2 = 0 instead. */
if (result2 != FLT_MIN) {
printf(
" FAILED: FTZ-only input not preserved: got %e, want %e "
"(line %d)\n",
(double) result2, (double) FLT_MIN, __LINE__);
ret = TEST_FAIL;
}
(void) result;

_mm_setcsr(original_csr);
return ret;
}

TEST_CASE(afp_daz_only_no_output_flush)
{
/* DAZ-only: denormal *outputs* must NOT be flushed to zero.
* Operation: FLT_MIN * 0.5 produces a denormal output. With FTZ that would
* be flushed to 0. With DAZ-only it must remain as the denormal value.
*/
unsigned int original_csr = _mm_getcsr();
result_t ret = TEST_SUCCESS;

float min_normal = FLT_MIN;
float half = 0.5f;
__m128 a = _mm_set1_ps(min_normal);
__m128 b = _mm_set1_ps(half);

/* Enable DAZ only, FTZ off */
_mm_setcsr(
(original_csr & ~static_cast<unsigned>(_MM_FLUSH_ZERO_MASK |
_MM_DENORMALS_ZERO_MASK)) |
_MM_DENORMALS_ZERO_ON);

__m128 c = _mm_mul_ps(a, b);
float result = extract_ps(c, 0);

/* FLT_MIN * 0.5 = FLT_MIN/2 (a denormal). The input FLT_MIN is normal so
* DAZ does not affect it. Without FTZ the denormal output must survive. */
float expected = make_half_flt_min();
if (result != expected) {
printf(
" FAILED: DAZ-only output flushed: got %e, want %e (line %d)\n",
(double) result, (double) expected, __LINE__);
ret = TEST_FAIL;
}

_mm_setcsr(original_csr);
return ret;
}
#endif /* __ARM_FEATURE_AFP */

/* Comparison Tests with Special Values */

TEST_CASE(cmpord_ps_nan)
Expand Down Expand Up @@ -1885,6 +2007,10 @@ int main(void)
RUN_TEST(daz_input_flush);
RUN_TEST(ftz_daz_disabled);
RUN_TEST(ftz_getcsr_roundtrip);
#if defined(__ARM_FEATURE_AFP)
RUN_TEST(afp_ftz_only_no_input_flush);
RUN_TEST(afp_daz_only_no_output_flush);
#endif

printf("\n--- Comparison Tests ---\n");
RUN_TEST(cmpord_ps_nan);
Expand Down
Loading