diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc index d453b0f1c1..cbc85b26c1 100644 --- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc +++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc @@ -113,7 +113,9 @@ R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); )w2c_template" R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +)w2c_template" +R"w2c_template( sizeof(t1)); \ )w2c_template" R"w2c_template( force_read(result); \ )w2c_template" @@ -169,7 +171,9 @@ R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ +)w2c_template" +R"w2c_template( sizeof(t1)); \ )w2c_template" R"w2c_template( } \ )w2c_template" @@ -207,45 +211,47 @@ R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ +#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( t2 value) { \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( t1 ret; \ +R"w2c_template( t1 ret; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ )w2c_template" -R"w2c_template( t1 newval = ret op wrapped; \ +R"w2c_template( t1 newval = ret op wrapped; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &newval, sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 ret = atomic_##opname( \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ +R"w2c_template( t1 ret = atomic_##opname( \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" @@ -325,43 +331,45 @@ R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ +#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( t2 value) { \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( t1 ret; \ +R"w2c_template( t1 ret; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 ret = atomic_##opname( \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ +R"w2c_template( t1 ret = atomic_##opname( \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" @@ -395,13 +403,13 @@ R"w2c_template( t2 replacement_wrapped = (t2)replacement; )w2c_template" R"w2c_template( t2 ret; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ )w2c_template" R"w2c_template( if (ret == expected_wrapped) { \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ )w2c_template" -R"w2c_template( sizeof(t2)); \ +R"w2c_template( &replacement_wrapped, sizeof(t2)); \ )w2c_template" R"w2c_template( } \ )w2c_template" diff --git a/src/prebuilt/wasm2c_simd_source_declarations.cc b/src/prebuilt/wasm2c_simd_source_declarations.cc index 2e70ecce05..c51139f586 100644 --- a/src/prebuilt/wasm2c_simd_source_declarations.cc +++ b/src/prebuilt/wasm2c_simd_source_declarations.cc @@ -19,60 +19,92 @@ R"w2c_template(#endif R"w2c_template(// TODO: equivalent constraint for ARM and other architectures )w2c_template" R"w2c_template( -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ +// The below SIMD operations copy to a local variable first as the )w2c_template" -R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template(// MEM_ADDR_MEMOP maybe segment pointers if WASM_RT_USE_SEGUE_FOR_THIS_MODULE is )w2c_template" -R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ +R"w2c_template(// defined and regular pointers otherwse. memcpy into the local works correctly )w2c_template" -R"w2c_template( SIMD_FORCE_READ(result); \ +R"w2c_template(// in both cases. )w2c_template" -R"w2c_template( return result; \ +R"w2c_template(#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +)w2c_template" +R"w2c_template( t simd_mem_value; \ +)w2c_template" +R"w2c_template( wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ +)w2c_template" +R"w2c_template( sizeof(t)); \ +)w2c_template" +R"w2c_template( v128 result = func(&simd_mem_value); \ +)w2c_template" +R"w2c_template( SIMD_FORCE_READ(result); \ +)w2c_template" +R"w2c_template( return result; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ +#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ +)w2c_template" +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( v128 vec) { \ )w2c_template" -R"w2c_template( v128 vec) { \ +R"w2c_template( t simd_mem_value; \ )w2c_template" -R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ +R"w2c_template( wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ )w2c_template" -R"w2c_template( SIMD_FORCE_READ(result); \ +R"w2c_template( sizeof(t)); \ )w2c_template" -R"w2c_template( return result; \ +R"w2c_template( v128 result = func(&simd_mem_value, vec, lane); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( SIMD_FORCE_READ(result); \ +)w2c_template" +R"w2c_template( return result; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE(name, t) \ +#define DEFINE_SIMD_STORE(name, t) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +)w2c_template" +R"w2c_template( v128 value) { \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( t simd_mem_value; \ )w2c_template" -R"w2c_template( v128 value) { \ +R"w2c_template( simde_wasm_v128_store(&simd_mem_value, value); \ )w2c_template" -R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( sizeof(t)); \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +)w2c_template" +R"w2c_template( v128 value) { \ +)w2c_template" +R"w2c_template( t simd_mem_value; \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( func(&simd_mem_value, value, lane); \ )w2c_template" -R"w2c_template( v128 value) { \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ )w2c_template" -R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ +R"w2c_template( sizeof(t)); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" diff --git a/src/template/wasm2c_atomicops.declarations.c b/src/template/wasm2c_atomicops.declarations.c index c2b767c9f2..4383c07cbb 100644 --- a/src/template/wasm2c_atomicops.declarations.c +++ b/src/template/wasm2c_atomicops.declarations.c @@ -59,7 +59,8 @@ DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ force_read(result); \ return (t3)(t2)result; \ } \ @@ -88,7 +89,8 @@ DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ + sizeof(t1)); \ } \ DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ @@ -108,26 +110,27 @@ DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64) DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) -#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ - static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ - t1 newval = ret op wrapped; \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &newval, sizeof(t1)); \ - return (t2)ret; \ - } \ - DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ - static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr, t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret = atomic_##opname( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ - return (t2)ret; \ - } \ +#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ + t1 newval = ret op wrapped; \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ + sizeof(t1)); \ + return (t2)ret; \ + } \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret = atomic_##opname( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ + return (t2)ret; \ + } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) @@ -170,25 +173,26 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) -#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ - static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ - return (t2)ret; \ - } \ - DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ - static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr, t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret = atomic_##opname( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ - return (t2)ret; \ - } \ +#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ + sizeof(t1)); \ + return (t2)ret; \ + } \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret = atomic_##opname( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ + return (t2)ret; \ + } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) @@ -206,10 +210,10 @@ DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) t2 expected_wrapped = (t2)expected; \ t2 replacement_wrapped = (t2)replacement; \ t2 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ if (ret == expected_wrapped) { \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ - sizeof(t2)); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ + &replacement_wrapped, sizeof(t2)); \ } \ return (t1)ret; \ } \ diff --git a/src/template/wasm2c_simd.declarations.c b/src/template/wasm2c_simd.declarations.c index abf3ce0f6e..01416c27b8 100644 --- a/src/template/wasm2c_simd.declarations.c +++ b/src/template/wasm2c_simd.declarations.c @@ -9,35 +9,51 @@ #endif // TODO: equivalent constraint for ARM and other architectures -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ - static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ - SIMD_FORCE_READ(result); \ - return result; \ - } \ +// The below SIMD operations copy to a local variable first as the +// MEM_ADDR_MEMOP maybe segment pointers if WASM_RT_USE_SEGUE_FOR_THIS_MODULE is +// defined and regular pointers otherwse. memcpy into the local works correctly +// in both cases. +#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t simd_mem_value; \ + wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ + sizeof(t)); \ + v128 result = func(&simd_mem_value); \ + SIMD_FORCE_READ(result); \ + return result; \ + } \ DEF_MEM_CHECKS0(name, _, t, return, v128); -#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ - static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 vec) { \ - v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ - SIMD_FORCE_READ(result); \ - return result; \ - } \ +#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 vec) { \ + t simd_mem_value; \ + wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ + sizeof(t)); \ + v128 result = func(&simd_mem_value, vec, lane); \ + SIMD_FORCE_READ(result); \ + return result; \ + } \ DEF_MEM_CHECKS1(name, _, t, return, v128, v128); -#define DEFINE_SIMD_STORE(name, t) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 value) { \ - simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ - } \ +#define DEFINE_SIMD_STORE(name, t) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + t simd_mem_value; \ + simde_wasm_v128_store(&simd_mem_value, value); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ + sizeof(t)); \ + } \ DEF_MEM_CHECKS1(name, _, t, , void, v128); -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 value) { \ - func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ - } \ +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + t simd_mem_value; \ + func(&simd_mem_value, value, lane); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ + sizeof(t)); \ + } \ DEF_MEM_CHECKS1(name, _, t, , void, v128); // clang-format off