diff --git a/src/c-writer.cc b/src/c-writer.cc index 93c469e3e8..ef4a2bd1aa 100644 --- a/src/c-writer.cc +++ b/src/c-writer.cc @@ -414,7 +414,10 @@ class CWriter { void WriteElemInitializers(); void WriteFuncRefWrappers(); void WriteElemTableInit(bool, const ElemSegment*, const Table*); + bool IsSingleUnsharedDefault32Memory(); bool IsSingleUnsharedMemory(); + void WriteLocalMemoryBaseSizeDeclaration(); + void RefreshLocalMemorySize(); void InstallSegueBase(Memory* memory, bool save_old_value); void RestoreSegueBase(); void WriteExports(CWriterPhase); @@ -1593,8 +1596,8 @@ void CWriter::WriteSourceTop() { Write(s_source_includes); Write(Newline(), "#include \"", header_name_, "\"", Newline()); - if (IsSingleUnsharedMemory()) { - Write("#define IS_SINGLE_UNSHARED_MEMORY 1", Newline()); + if (IsSingleUnsharedDefault32Memory()) { + Write("#define IS_SINGLE_UNSHARED_DEFAULT32_MEMORY 1", Newline()); } Write(s_source_declarations, Newline()); @@ -2612,9 +2615,41 @@ void CWriter::WriteElemTableInit(bool active_initialization, Write(");", Newline()); } -bool CWriter::IsSingleUnsharedMemory() { +bool CWriter::IsSingleUnsharedDefault32Memory() { return module_->memories.size() == 1 && - !module_->memories[0]->page_limits.is_shared; + !module_->memories[0]->page_limits.is_shared && + module_->memories[0]->page_size == WABT_DEFAULT_PAGE_SIZE && + !module_->memories[0]->page_limits.is_64; +} + +void CWriter::WriteLocalMemoryBaseSizeDeclaration() { + Write("uint8_t* const wasm_rt_local_memory_base = "); + if (IsSingleUnsharedDefault32Memory()) { + const Memory* memory = module_->memories[0]; + Write("(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name), + ")->data"); + } else { + Write("NULL"); + } + Write(";", Newline(), "uint64_t wasm_rt_local_memory_size = "); + if (IsSingleUnsharedDefault32Memory()) { + const Memory* memory = module_->memories[0]; + Write("(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name), + ")->size"); + } else { + Write("0"); + } + Write(";", Newline(), "(void)wasm_rt_local_memory_base;", Newline(), + "(void)wasm_rt_local_memory_size;", Newline()); +} + +void CWriter::RefreshLocalMemorySize() { + if (IsSingleUnsharedDefault32Memory()) { + const Memory* memory = module_->memories[0]; + Write("wasm_rt_local_memory_size = (", + ExternalInstancePtr(ModuleFieldType::Memory, memory->name), + ")->size;", Newline()); + } } void CWriter::InstallSegueBase(Memory* memory, bool save_old_value) { @@ -2716,7 +2751,7 @@ void CWriter::WriteExports(CWriterPhase kind) { switch (export_->kind) { case ExternalKind::Func: { Write(OpenBrace()); - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { InstallSegueBase(module_->memories[0], true /* save_old_value */); } auto num_results = func_->GetNumResults(); @@ -2735,7 +2770,7 @@ void CWriter::WriteExports(CWriterPhase kind) { Write("instance"); } WriteParamSymbols(index_to_name); - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { RestoreSegueBase(); } if (num_results > 0) { @@ -2839,7 +2874,7 @@ void CWriter::WriteInit() { } if (!module_->memories.empty()) { Write("init_memories(instance);", Newline()); - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { InstallSegueBase(module_->memories[0], true /* save_old_value */); } } @@ -2863,7 +2898,7 @@ void CWriter::WriteInit() { Write(Newline()); } - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { RestoreSegueBase(); } Write(CloseBrace(), Newline()); @@ -3188,6 +3223,7 @@ void CWriter::Write(const Func& func) { GlobalName(ModuleFieldType::Func, func.name), "("); WriteParamsAndLocals(); Write("FUNC_PROLOGUE;", Newline()); + WriteLocalMemoryBaseSizeDeclaration(); size_t stack_vars_section = func_sections_.size() - 1; PushFuncSection(); @@ -3262,6 +3298,7 @@ void CWriter::WriteTailCallee(const Func& func) { Write(" ", OpenBrace()); WriteTailCallAsserts(func.decl.sig); Write(ModuleInstanceTypeName(), "* instance = *instance_ptr;", Newline()); + WriteLocalMemoryBaseSizeDeclaration(); std::vector index_to_name; MakeTypeBindingReverseMapping(func.GetNumParamsAndLocals(), func.bindings, @@ -3430,6 +3467,7 @@ size_t CWriter::BeginTry(const Block& block) { void CWriter::WriteTryCatch(const TryExpr& tryexpr) { const size_t mark = BeginTry(tryexpr.block); + RefreshLocalMemorySize(); /* exception has been thrown -- do we catch it? */ @@ -3576,6 +3614,7 @@ void CWriter::WriteTryDelegate(const TryExpr& tryexpr) { void CWriter::Write(const TryTableExpr& try_table_expr) { const size_t mark = BeginTry(try_table_expr.block); + RefreshLocalMemorySize(); /* exception has been thrown -- do we catch it? */ @@ -3730,6 +3769,7 @@ void CWriter::Write(const ExprList& exprs) { Write(StackVar(num_params - i - 1)); } Write(");", Newline()); + RefreshLocalMemorySize(); DropTypes(num_params); PushTypes(func.decl.sig.result_types); if (num_results > 1) { @@ -3766,9 +3806,10 @@ void CWriter::Write(const ExprList& exprs) { Write(", ", StackVar(num_params - i)); } Write(");", Newline()); - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { InstallSegueBase(module_->memories[0], false /* save_old_value */); } + RefreshLocalMemorySize(); DropTypes(num_params + 1); PushTypes(decl.sig.result_types); if (num_results > 1) { @@ -4094,9 +4135,10 @@ void CWriter::Write(const ExprList& exprs) { Write(StackVar(0), " = ", func, "(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", ", StackVar(0), ");", Newline()); - if (IsSingleUnsharedMemory()) { + if (IsSingleUnsharedDefault32Memory()) { InstallSegueBase(module_->memories[0], false /* save_old_value */); } + RefreshLocalMemorySize(); break; } @@ -5386,8 +5428,9 @@ void CWriter::Write(const LoadExpr& expr) { func = GetMemoryAPIString(*memory, func); Type result_type = expr.opcode.GetResultType(); - Write(StackVar(0, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(0, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(0, memory, expr.offset); Write(");", Newline()); DropTypes(1); @@ -5417,8 +5460,8 @@ void CWriter::Write(const StoreExpr& expr) { Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)]; func = GetMemoryAPIString(*memory, func); - Write(func, "(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name), - ", "); + Write(func, "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(1, memory, expr.offset); Write(", ", StackVar(0), ");", Newline()); DropTypes(2); @@ -5876,8 +5919,9 @@ void CWriter::Write(const SimdLoadLaneExpr& expr) { // clang-format on Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)]; Type result_type = expr.opcode.GetResultType(); - Write(StackVar(1, result_type), " = ", func, expr.val, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(1, result_type), " = ", func, expr.val, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(1, memory, expr.offset); Write(", ", StackVar(0)); Write(");", Newline()); @@ -5900,8 +5944,9 @@ void CWriter::Write(const SimdStoreLaneExpr& expr) { // clang-format on Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)]; - Write(func, expr.val, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(func, expr.val, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(1, memory, expr.offset); Write(", ", StackVar(0)); Write(");", Newline()); @@ -5943,8 +5988,9 @@ void CWriter::Write(const LoadSplatExpr& expr) { } // clang-format on Type result_type = expr.opcode.GetResultType(); - Write(StackVar(0, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(0, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(0, memory, expr.offset); Write(");", Newline()); @@ -5966,8 +6012,9 @@ void CWriter::Write(const LoadZeroExpr& expr) { // clang-format on Type result_type = expr.opcode.GetResultType(); - Write(StackVar(0, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(0, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(0, memory, expr.offset); Write(");", Newline()); @@ -5996,8 +6043,9 @@ void CWriter::Write(const AtomicLoadExpr& expr) { func = GetMemoryAPIString(*memory, func); Type result_type = expr.opcode.GetResultType(); - Write(StackVar(0, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(0, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(0, memory, expr.offset); Write(");", Newline()); DropTypes(1); @@ -6024,8 +6072,8 @@ void CWriter::Write(const AtomicStoreExpr& expr) { Memory* memory = module_->memories[module_->GetMemoryIndex(expr.memidx)]; func = GetMemoryAPIString(*memory, func); - Write(func, "(", ExternalInstancePtr(ModuleFieldType::Memory, memory->name), - ", "); + Write(func, "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(1, memory, expr.offset); Write(", ", StackVar(0), ");", Newline()); DropTypes(2); @@ -6087,8 +6135,9 @@ void CWriter::Write(const AtomicRmwExpr& expr) { Type result_type = expr.opcode.GetResultType(); - Write(StackVar(1, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(1, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(1, memory, expr.offset); Write(", ", StackVar(0), ");", Newline()); DropTypes(2); @@ -6116,8 +6165,9 @@ void CWriter::Write(const AtomicRmwCmpxchgExpr& expr) { Type result_type = expr.opcode.GetResultType(); - Write(StackVar(2, result_type), " = ", func, "(", - ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); + Write(StackVar(2, result_type), " = ", func, + "(wasm_rt_local_memory_base, wasm_rt_local_memory_size, "); + Write(ExternalInstancePtr(ModuleFieldType::Memory, memory->name), ", "); WriteMemoryAddress(2, memory, expr.offset); Write(", ", StackVar(1), ", ", StackVar(0), ");", Newline()); DropTypes(3); diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc index 25c122a234..0bfe0cd1e5 100644 --- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc +++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc @@ -19,7 +19,9 @@ R"w2c_template( } R"w2c_template( #define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ +)w2c_template" +R"w2c_template( wasm_rt_shared_memory_t* mem, u64 addr) { \ )w2c_template" R"w2c_template( t1 result; \ )w2c_template" @@ -71,9 +73,11 @@ R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ R"w2c_template( #define DEFINE_SHARED_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( static inline void name##_unchecked( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" @@ -107,47 +111,51 @@ R"w2c_template(DEFINE_SHARED_STORE(i64_store16_shared, u16, u64) R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ +#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ )w2c_template" -R"w2c_template( t3 ret = (t3)(t2)result; \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( force_read(ret); \ +R"w2c_template( t3 ret = (t3)(t2)result; \ )w2c_template" -R"w2c_template( return ret; \ +R"w2c_template( force_read(ret); \ +)w2c_template" +R"w2c_template( return ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) \ +R"w2c_template( static inline t3 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( u64 addr) { \ +R"w2c_template( u64 addr) { \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( result = \ +R"w2c_template( result = \ )w2c_template" -R"w2c_template( atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ +R"w2c_template( atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ )w2c_template" -R"w2c_template( t3 ret = (t3)(t2)result; \ +R"w2c_template( t3 ret = (t3)(t2)result; \ )w2c_template" -R"w2c_template( force_read(ret); \ +R"w2c_template( force_read(ret); \ )w2c_template" -R"w2c_template( return ret; \ +R"w2c_template( return ret; \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) )w2c_template" @@ -167,37 +175,41 @@ R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_STORE(name, t1, t2) \ +#define DEFINE_ATOMIC_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline void name##_unchecked( \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ +R"w2c_template( t1 wrapped = (t1)value; \ +)w2c_template" +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( sizeof(t1)); \ +)w2c_template" +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ +R"w2c_template( static inline void name##_shared_unchecked( \ )w2c_template" -R"w2c_template( static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( u64 addr, t2 value) { \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ +R"w2c_template( atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ )w2c_template" -R"w2c_template( wrapped); \ +R"w2c_template( wrapped); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) )w2c_template" @@ -217,47 +229,51 @@ R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ +#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t2 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 ret; \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ +R"w2c_template( t1 ret; \ )w2c_template" -R"w2c_template( t1 newval = ret op wrapped; \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ +R"w2c_template( t1 newval = ret op wrapped; \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( t1 ret = atomic_##opname( \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( t1 ret = atomic_##opname( \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ +)w2c_template" +R"w2c_template( return (t2)ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" @@ -337,45 +353,49 @@ R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ +#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ +)w2c_template" +R"w2c_template( static inline t2 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( t2 value) { \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( t1 ret; \ +R"w2c_template( t1 ret; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +R"w2c_template( static inline t2 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( u64 addr, t2 value) { \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" -R"w2c_template( t1 wrapped = (t1)value; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" -R"w2c_template( t1 ret = atomic_##opname( \ +R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ +R"w2c_template( t1 ret = atomic_##opname( \ )w2c_template" -R"w2c_template( return (t2)ret; \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t2)ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" @@ -395,55 +415,59 @@ R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t1 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( t1 expected, t1 replacement) { \ +R"w2c_template( wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( t2 ret; \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ +R"w2c_template( t2 ret; \ )w2c_template" -R"w2c_template( if (ret == expected_wrapped) { \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ +R"w2c_template( if (ret == expected_wrapped) { \ )w2c_template" -R"w2c_template( &replacement_wrapped, sizeof(t2)); \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( &replacement_wrapped, sizeof(t2)); \ )w2c_template" -R"w2c_template( return (t1)ret; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t1)ret; \ )w2c_template" -R"w2c_template( DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ )w2c_template" -R"w2c_template( static inline t1 name##_shared_unchecked( \ +R"w2c_template( static inline t1 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( u64 addr, t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( atomic_compare_exchange_strong( \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ +R"w2c_template( atomic_compare_exchange_strong( \ )w2c_template" -R"w2c_template( &expected_wrapped, replacement_wrapped); \ +R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ )w2c_template" -R"w2c_template( return (t1)expected_wrapped; \ +R"w2c_template( &expected_wrapped, replacement_wrapped); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t1)expected_wrapped; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) )w2c_template" diff --git a/src/prebuilt/wasm2c_simd_source_declarations.cc b/src/prebuilt/wasm2c_simd_source_declarations.cc index 942e3e48d6..53e4b5e3f0 100644 --- a/src/prebuilt/wasm2c_simd_source_declarations.cc +++ b/src/prebuilt/wasm2c_simd_source_declarations.cc @@ -41,7 +41,11 @@ R"w2c_template(// in both cases. )w2c_template" R"w2c_template(#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ )w2c_template" -R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline v128 name##_unchecked( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr) { \ )w2c_template" R"w2c_template( t simd_mem_value; \ )w2c_template" @@ -62,9 +66,11 @@ R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128); R"w2c_template( #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ )w2c_template" -R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline v128 name##_unchecked( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ )w2c_template" -R"w2c_template( v128 vec) { \ +R"w2c_template( u64 addr, v128 vec) { \ )w2c_template" R"w2c_template( t simd_mem_value; \ )w2c_template" @@ -85,9 +91,11 @@ R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128); R"w2c_template( #define DEFINE_SIMD_STORE(name, t) \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline void name##_unchecked( \ )w2c_template" -R"w2c_template( v128 value) { \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr, v128 value) { \ )w2c_template" R"w2c_template( t simd_mem_value; \ )w2c_template" @@ -104,9 +112,11 @@ R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); R"w2c_template( #define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline void name##_unchecked( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ )w2c_template" -R"w2c_template( v128 value) { \ +R"w2c_template( u64 addr, v128 value) { \ )w2c_template" R"w2c_template( t simd_mem_value; \ )w2c_template" diff --git a/src/prebuilt/wasm2c_source_declarations.cc b/src/prebuilt/wasm2c_source_declarations.cc index d6b6cd8e0c..67f33db855 100644 --- a/src/prebuilt/wasm2c_source_declarations.cc +++ b/src/prebuilt/wasm2c_source_declarations.cc @@ -40,11 +40,11 @@ R"w2c_template(#define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) R"w2c_template(#endif )w2c_template" R"w2c_template( -// We can only use Segue for this module if it uses a single unshared imported +// We can only use Segue for this module if it uses a single unshared, )w2c_template" -R"w2c_template(// or exported memory +R"w2c_template(// default-page, 32-bit imported or exported memory. )w2c_template" -R"w2c_template(#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +R"w2c_template(#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY )w2c_template" R"w2c_template(#define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 )w2c_template" @@ -55,6 +55,26 @@ R"w2c_template(#define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 R"w2c_template(#endif )w2c_template" R"w2c_template( +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +)w2c_template" +R"w2c_template(#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +)w2c_template" +R"w2c_template(#endif +)w2c_template" +R"w2c_template( +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ +)w2c_template" +R"w2c_template( IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +)w2c_template" +R"w2c_template(#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +)w2c_template" +R"w2c_template(#else +)w2c_template" +R"w2c_template(#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +)w2c_template" +R"w2c_template(#endif +)w2c_template" +R"w2c_template( #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE )w2c_template" R"w2c_template(// POSIX uses FS for TLS, GS is free @@ -103,6 +123,10 @@ R"w2c_template(} )w2c_template" R"w2c_template(#define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) )w2c_template" +R"w2c_template(#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +)w2c_template" +R"w2c_template(#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) +)w2c_template" R"w2c_template(#else )w2c_template" R"w2c_template(#define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) @@ -283,7 +307,9 @@ R"w2c_template(// or it may do a slightly faster RANGE_CHECK. )w2c_template" R"w2c_template(#if WASM_RT_MEMCHECK_GUARD_PAGES )w2c_template" -R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ +)w2c_template" +R"w2c_template( WASM_RT_CHECK_BASE(mem); )w2c_template" R"w2c_template( // When using guard pages, reads have to be immediately consumed so that OOB @@ -355,16 +381,30 @@ R"w2c_template(#endif R"w2c_template( #else )w2c_template" -R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) \ +R"w2c_template(#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +)w2c_template" +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ +)w2c_template" +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +)w2c_template" +R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ +)w2c_template" +R"w2c_template( TRAP(OOB); +)w2c_template" +R"w2c_template(#else +)w2c_template" +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ )w2c_template" -R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ )w2c_template" -R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ )w2c_template" R"w2c_template( TRAP(OOB); )w2c_template" R"w2c_template(#endif )w2c_template" +R"w2c_template(#endif +)w2c_template" R"w2c_template( // MEMCHECK_GENERAL can be used for any memory )w2c_template" @@ -427,103 +467,129 @@ R"w2c_template( } while (0) R"w2c_template( #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ )w2c_template" -R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +R"w2c_template( static inline return_type name##_default32( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, \ +)w2c_template" +R"w2c_template( uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ )w2c_template" -R"w2c_template( u64 addr) { \ +R"w2c_template( u64 addr) { \ )w2c_template" -R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +R"w2c_template( MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ +R"w2c_template( static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ +)w2c_template" +R"w2c_template( uint64_t wasm_rt_local_memory_size, \ +)w2c_template" +R"w2c_template( wasm_rt##shared##memory_t* mem, u64 addr) { \ )w2c_template" R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ )w2c_template" R"w2c_template( } )w2c_template" R"w2c_template( -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ )w2c_template" -R"w2c_template( val_type1) \ +R"w2c_template( val_type1) \ )w2c_template" -R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +R"w2c_template( static inline return_type name##_default32( \ )w2c_template" -R"w2c_template( u64 addr, val_type1 val1) { \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +R"w2c_template( uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +R"w2c_template( u64 addr, val_type1 val1) { \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ )w2c_template" -R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ )w2c_template" -R"w2c_template( val_type1 val1) { \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +R"w2c_template( static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ +)w2c_template" +R"w2c_template( uint64_t wasm_rt_local_memory_size, \ +)w2c_template" +R"w2c_template( wasm_rt##shared##memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +R"w2c_template( val_type1 val1) { \ +)w2c_template" +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ )w2c_template" R"w2c_template( } )w2c_template" R"w2c_template( -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ )w2c_template" -R"w2c_template( val_type1, val_type2) \ +R"w2c_template( val_type1, val_type2) \ )w2c_template" -R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +R"w2c_template( static inline return_type name##_default32( \ )w2c_template" -R"w2c_template( u64 addr, val_type1 val1, \ +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( val_type2 val2) { \ +R"w2c_template( uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +R"w2c_template( u64 addr, val_type1 val1, val_type2 val2) { \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ +R"w2c_template( MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ )w2c_template" -R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( val_type1 val1, val_type2 val2) { \ +R"w2c_template( static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +R"w2c_template( uint64_t wasm_rt_local_memory_size, \ +)w2c_template" +R"w2c_template( wasm_rt##shared##memory_t* mem, u64 addr, \ +)w2c_template" +R"w2c_template( val_type1 val1, val_type2 val2) { \ )w2c_template" -R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ )w2c_template" R"w2c_template( } )w2c_template" R"w2c_template( -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ )w2c_template" -R"w2c_template( t3 ret = (t3)(t2)result; \ +R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( force_read(ret); \ +R"w2c_template( t3 ret = (t3)(t2)result; \ )w2c_template" -R"w2c_template( return ret; \ +R"w2c_template( force_read(ret); \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return ret; \ +)w2c_template" +R"w2c_template( } \ )w2c_template" R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) )w2c_template" R"w2c_template( #define DEFINE_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ +R"w2c_template( static inline void name##_unchecked( \ +)w2c_template" +R"w2c_template( uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" diff --git a/src/template/wasm2c.declarations.c b/src/template/wasm2c.declarations.c index c24364f655..6e5b92fb2b 100644 --- a/src/template/wasm2c.declarations.c +++ b/src/template/wasm2c.declarations.c @@ -20,14 +20,25 @@ #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -53,6 +64,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -150,7 +163,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -188,10 +202,17 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); +#endif #endif // MEMCHECK_GENERAL can be used for any memory @@ -227,57 +248,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } - -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } - -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } - -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ diff --git a/src/template/wasm2c_atomicops.declarations.c b/src/template/wasm2c_atomicops.declarations.c index b4088daade..d3ea511f40 100644 --- a/src/template/wasm2c_atomicops.declarations.c +++ b/src/template/wasm2c_atomicops.declarations.c @@ -10,7 +10,8 @@ } #define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_shared_memory_t* mem, u64 addr) { \ t1 result; \ result = atomic_load_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ @@ -37,8 +38,9 @@ DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT) #define DEFINE_SHARED_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ atomic_store_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped, \ @@ -56,27 +58,29 @@ DEFINE_SHARED_STORE(i64_store8_shared, u8, u64) DEFINE_SHARED_STORE(i64_store16_shared, u16, u64) DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) -#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ - DEF_MEM_CHECKS0(name, _, t1, return, t3) \ - static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 result; \ - result = \ - atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ +#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) \ + static inline t3 name##_shared_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 result; \ + result = \ + atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) @@ -87,22 +91,24 @@ DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) -#define DEFINE_ATOMIC_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ - sizeof(t1)); \ - } \ - DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ - static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr, t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ - wrapped); \ - } \ +#define DEFINE_ATOMIC_STORE(name, t1, t2) \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ + sizeof(t1)); \ + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ + static inline void name##_shared_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ + wrapped); \ + } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) @@ -113,27 +119,29 @@ DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64) DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) -#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ - static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ - t1 newval = ret op wrapped; \ - wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ - sizeof(t1)); \ - return (t2)ret; \ - } \ - DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ - static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr, t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret = atomic_##opname( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ - return (t2)ret; \ - } \ +#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ + static inline t2 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ + t1 newval = ret op wrapped; \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &newval, \ + sizeof(t1)); \ + return (t2)ret; \ + } \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret = atomic_##opname( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ + return (t2)ret; \ + } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) @@ -176,26 +184,28 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) -#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ - static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ - wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ - sizeof(t1)); \ - return (t2)ret; \ - } \ - DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ - static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ - u64 addr, t2 value) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - t1 ret = atomic_##opname( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ - return (t2)ret; \ - } \ +#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ + static inline t2 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), sizeof(t1)); \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ + sizeof(t1)); \ + return (t2)ret; \ + } \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + t1 ret = atomic_##opname( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ + return (t2)ret; \ + } \ DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) @@ -206,31 +216,33 @@ DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ - static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t1 expected, t1 replacement) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - t2 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ - if (ret == expected_wrapped) { \ - wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ - &replacement_wrapped, sizeof(t2)); \ - } \ - return (t1)ret; \ - } \ - DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ - static inline t1 name##_shared_unchecked( \ - wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - atomic_compare_exchange_strong( \ - (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ - &expected_wrapped, replacement_wrapped); \ - return (t1)expected_wrapped; \ - } \ +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ + static inline t1 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr, \ + t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + t2 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), sizeof(t2)); \ + if (ret == expected_wrapped) { \ + wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t2)), \ + &replacement_wrapped, sizeof(t2)); \ + } \ + return (t1)ret; \ + } \ + DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ + static inline t1 name##_shared_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_shared_memory_t* mem, \ + u64 addr, t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + atomic_compare_exchange_strong( \ + (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ + &expected_wrapped, replacement_wrapped); \ + return (t1)expected_wrapped; \ + } \ DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); diff --git a/src/template/wasm2c_simd.declarations.c b/src/template/wasm2c_simd.declarations.c index 3a5c3f544a..84bd54aa4e 100644 --- a/src/template/wasm2c_simd.declarations.c +++ b/src/template/wasm2c_simd.declarations.c @@ -21,7 +21,9 @@ // defined and regular pointers otherwse. memcpy into the local works correctly // in both cases. #define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ - static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + static inline v128 name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr) { \ t simd_mem_value; \ wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ sizeof(t)); \ @@ -32,8 +34,9 @@ DEF_MEM_CHECKS0(name, _, t, return, v128); #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ - static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 vec) { \ + static inline v128 name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, v128 vec) { \ t simd_mem_value; \ wasm_rt_memcpy(&simd_mem_value, MEM_ADDR_MEMOP(mem, addr, sizeof(t)), \ sizeof(t)); \ @@ -44,8 +47,9 @@ DEF_MEM_CHECKS1(name, _, t, return, v128, v128); #define DEFINE_SIMD_STORE(name, t) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, v128 value) { \ t simd_mem_value; \ simde_wasm_v128_store(&simd_mem_value, value); \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ @@ -54,8 +58,9 @@ DEF_MEM_CHECKS1(name, _, t, , void, v128); #define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - v128 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, v128 value) { \ t simd_mem_value; \ func(&simd_mem_value, value, lane); \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t)), &simd_mem_value, \ diff --git a/test/wasm2c/add.txt b/test/wasm2c/add.txt index 18d0755a41..ff36819b56 100644 --- a/test/wasm2c/add.txt +++ b/test/wasm2c/add.txt @@ -87,14 +87,25 @@ u32 w2c_test_add(w2c_test*, u32, u32); #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -120,6 +131,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -217,7 +230,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -255,10 +269,17 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); +#endif #endif // MEMCHECK_GENERAL can be used for any memory @@ -294,57 +315,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } - -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } - -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } - -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ @@ -901,6 +935,10 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res u32 w2c_test_add_0(w2c_test* instance, u32 var_p0, u32 var_p1) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = NULL; + uint64_t wasm_rt_local_memory_size = 0; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0, var_i1; var_i0 = var_p0; var_i1 = var_p1; diff --git a/test/wasm2c/check-imports.txt b/test/wasm2c/check-imports.txt index 176032da45..b1c5b9558d 100644 --- a/test/wasm2c/check-imports.txt +++ b/test/wasm2c/check-imports.txt @@ -89,7 +89,7 @@ extern const u32 wasm2c_test_pagesize_env_0x5F_linear_memory; #endif #include "wasm.h" -#define IS_SINGLE_UNSHARED_MEMORY 1 +#define IS_SINGLE_UNSHARED_DEFAULT32_MEMORY 1 // Computes a pointer to an object of the given size in a little-endian memory. // @@ -112,14 +112,25 @@ extern const u32 wasm2c_test_pagesize_env_0x5F_linear_memory; #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -145,6 +156,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -242,7 +255,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -280,11 +294,18 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ + TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ TRAP(OOB); #endif +#endif // MEMCHECK_GENERAL can be used for any memory #define MEMCHECK_GENERAL(mem, a, t) \ @@ -319,57 +340,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ } -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ } -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ @@ -985,30 +1019,44 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res u32 w2c_test_f0(w2c_test* instance, u32 var_p0) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = (instance->w2c_env_0x5F_linear_memory)->data; + uint64_t wasm_rt_local_memory_size = (instance->w2c_env_0x5F_linear_memory)->size; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0; var_i0 = var_p0; var_i0 = CALL_INDIRECT((*instance->w2c_env_0x5F_indirect_function_table), u32 (*)(void*), w2c_test_t1, var_i0, (*instance->w2c_env_0x5F_indirect_function_table).data[var_i0].module_instance); #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE wasm_rt_segue_write_base((*instance->w2c_env_0x5F_linear_memory).data); #endif + wasm_rt_local_memory_size = (instance->w2c_env_0x5F_linear_memory)->size; FUNC_EPILOGUE; return var_i0; } u32 w2c_test_f1(w2c_test* instance) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = (instance->w2c_env_0x5F_linear_memory)->data; + uint64_t wasm_rt_local_memory_size = (instance->w2c_env_0x5F_linear_memory)->size; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0; var_i0 = 16u; - var_i0 = i32_load_default32(instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); + var_i0 = i32_load_default32(wasm_rt_local_memory_base, wasm_rt_local_memory_size, instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); FUNC_EPILOGUE; return var_i0; } u32 w2c_test_f2(w2c_test* instance, u32 var_p0, u32 var_p1) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = (instance->w2c_env_0x5F_linear_memory)->data; + uint64_t wasm_rt_local_memory_size = (instance->w2c_env_0x5F_linear_memory)->size; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0; var_i0 = 1u; var_i0 = w2c_test_f0(instance, var_i0); + wasm_rt_local_memory_size = (instance->w2c_env_0x5F_linear_memory)->size; FUNC_EPILOGUE; return var_i0; } diff --git a/test/wasm2c/export-names.txt b/test/wasm2c/export-names.txt index 332a26c087..775054988a 100644 --- a/test/wasm2c/export-names.txt +++ b/test/wasm2c/export-names.txt @@ -89,7 +89,7 @@ void w2c_test_0xE20x9D0xA40xEF0xB80x8F(w2c_test*); #endif #include "wasm.h" -#define IS_SINGLE_UNSHARED_MEMORY 1 +#define IS_SINGLE_UNSHARED_DEFAULT32_MEMORY 1 // Computes a pointer to an object of the given size in a little-endian memory. // @@ -112,14 +112,25 @@ void w2c_test_0xE20x9D0xA40xEF0xB80x8F(w2c_test*); #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -145,6 +156,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -242,7 +255,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -280,11 +294,18 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ + TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ TRAP(OOB); #endif +#endif // MEMCHECK_GENERAL can be used for any memory #define MEMCHECK_GENERAL(mem, a, t) \ @@ -319,57 +340,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ } -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ } -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ @@ -1013,6 +1047,10 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res void w2c_test__0(w2c_test* instance) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = (instance->w2c_0x5Cmodule_import0x200x2A0x2F)->data; + uint64_t wasm_rt_local_memory_size = (instance->w2c_0x5Cmodule_import0x200x2A0x2F)->size; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; FUNC_EPILOGUE; } ;;; STDOUT ;;) diff --git a/test/wasm2c/hello.txt b/test/wasm2c/hello.txt index ee5bd4ea42..ce5b697cf0 100644 --- a/test/wasm2c/hello.txt +++ b/test/wasm2c/hello.txt @@ -96,7 +96,7 @@ void w2c_test_0x5Fstart(w2c_test*); #endif #include "wasm.h" -#define IS_SINGLE_UNSHARED_MEMORY 1 +#define IS_SINGLE_UNSHARED_DEFAULT32_MEMORY 1 // Computes a pointer to an object of the given size in a little-endian memory. // @@ -119,14 +119,25 @@ void w2c_test_0x5Fstart(w2c_test*); #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -152,6 +163,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -249,7 +262,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -287,11 +301,18 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ + TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ TRAP(OOB); #endif +#endif // MEMCHECK_GENERAL can be used for any memory #define MEMCHECK_GENERAL(mem, a, t) \ @@ -326,57 +347,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ } -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ } -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ @@ -1015,13 +1049,17 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res void w2c_test_0x5Fstart_0(w2c_test* instance) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = (&instance->w2c_memory)->data; + uint64_t wasm_rt_local_memory_size = (&instance->w2c_memory)->size; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0, var_i1, var_i2, var_i3, var_i4; var_i0 = 0u; var_i1 = 8u; - i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(wasm_rt_local_memory_base, wasm_rt_local_memory_size, &instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 4u; var_i1 = 14u; - i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(wasm_rt_local_memory_base, wasm_rt_local_memory_size, &instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 1u; var_i1 = 0u; var_i2 = 1u; @@ -1031,7 +1069,9 @@ void w2c_test_0x5Fstart_0(w2c_test* instance) { #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE wasm_rt_segue_write_base(instance->w2c_memory.data); #endif + wasm_rt_local_memory_size = (&instance->w2c_memory)->size; w2c_wasi__snapshot__preview1_proc_exit(instance->w2c_wasi__snapshot__preview1_instance, var_i0); + wasm_rt_local_memory_size = (&instance->w2c_memory)->size; FUNC_EPILOGUE; } ;;; STDOUT ;;) diff --git a/test/wasm2c/minimal.txt b/test/wasm2c/minimal.txt index 95c3a3132a..437832bf57 100644 --- a/test/wasm2c/minimal.txt +++ b/test/wasm2c/minimal.txt @@ -81,14 +81,25 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -114,6 +125,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -211,7 +224,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -249,10 +263,17 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); +#endif #endif // MEMCHECK_GENERAL can be used for any memory @@ -288,57 +309,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ - } - -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } - -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } - -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ diff --git a/test/wasm2c/tail-calls.txt b/test/wasm2c/tail-calls.txt index edf91f0928..b7e1f3664c 100644 --- a/test/wasm2c/tail-calls.txt +++ b/test/wasm2c/tail-calls.txt @@ -111,14 +111,25 @@ void wasm_tailcall_w2c_test_tailcaller(void **instance_ptr, void *tail_call_stac #define MEM_ADDR(mem, addr, n) &((mem)->data[addr]) #endif -// We can only use Segue for this module if it uses a single unshared imported -// or exported memory -#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY +// We can only use Segue for this module if it uses a single unshared, +// default-page, 32-bit imported or exported memory. +#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_DEFAULT32_MEMORY #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1 #else #define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0 #endif +#ifndef WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE 1 +#endif + +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE && WASM_RT_USE_MMAP && \ + IS_SINGLE_UNSHARED_DEFAULT32_MEMORY && !WABT_BIG_ENDIAN +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 1 +#else +#define WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE 0 +#endif + #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE // POSIX uses FS for TLS, GS is free static inline void* wasm_rt_segue_read_base() { @@ -144,6 +155,8 @@ static inline void wasm_rt_segue_write_base(void* base) { } } #define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr) +#elif WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEM_ADDR_MEMOP(mem, addr, n) (&wasm_rt_local_memory_base[(addr)]) #else #define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n) #endif @@ -241,7 +254,8 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { // (if hardware bounds-checking is enabled via guard pages) // or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); // When using guard pages, reads have to be immediately consumed so that OOB // trap checks are applied in the right place, and not optimized away. @@ -279,11 +293,18 @@ static inline uint64_t checked_add_u64(uint64_t a, uint64_t b) { #endif #else -#define MEMCHECK_DEFAULT32(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +#if WASM_RT_USE_LOCAL_MEMORY_BASE_SIZE_FOR_THIS_MODULE +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > local_memory_size)) \ + TRAP(OOB); +#else +#define MEMCHECK_DEFAULT32(mem, local_memory_size, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ TRAP(OOB); #endif +#endif // MEMCHECK_GENERAL can be used for any memory #define MEMCHECK_GENERAL(mem, a, t) \ @@ -318,57 +339,70 @@ static inline void load_data(u8* dest, const u8* src, size_t n) { } while (0) #define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr) { \ MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr); \ } -#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ - val_type1) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1); \ +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1); \ } -#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ - val_type1, val_type2) \ - static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ - u64 addr, val_type1 val1, \ - val_type2 val2) { \ - MEMCHECK_DEFAULT32(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ - } \ - static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ - val_type1 val1, val_type2 val2) { \ - MEMCHECK_GENERAL(mem, addr, mem_type); \ - ret_kw name##_unchecked(mem, addr, val1, val2); \ +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32( \ + uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, wasm_rt_local_memory_size, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ + } \ + static inline return_type name(uint8_t* const wasm_rt_local_memory_base, \ + uint64_t wasm_rt_local_memory_size, \ + wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(wasm_rt_local_memory_base, mem, addr, val1, val2); \ } -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - t3 ret = (t3)(t2)result; \ - force_read(ret); \ - return ret; \ - } \ +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(uint8_t* const wasm_rt_local_memory_base, \ + wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + t3 ret = (t3)(t2)result; \ + force_read(ret); \ + return ret; \ + } \ DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ - t2 value) { \ + static inline void name##_unchecked( \ + uint8_t* const wasm_rt_local_memory_base, wasm_rt_memory_t* mem, \ + u64 addr, t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ @@ -993,6 +1027,10 @@ wasm_rt_func_type_t wasm2c_test_get_func_type(uint32_t param_count, uint32_t res void w2c_test_tailcaller_0(w2c_test* instance) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = NULL; + uint64_t wasm_rt_local_memory_size = 0; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0, var_i2; f32 var_f1; var_i0 = 1u; @@ -1025,6 +1063,10 @@ void w2c_test_tailcaller_0(w2c_test* instance) { void wasm_tailcall_w2c_test_tailcaller_0(void **instance_ptr, void *tail_call_stack, wasm_rt_tailcallee_t *next) { w2c_test* instance = *instance_ptr; + uint8_t* const wasm_rt_local_memory_base = NULL; + uint64_t wasm_rt_local_memory_size = 0; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0, var_i2; f32 var_f1; var_i0 = 1u; @@ -1052,6 +1094,10 @@ void wasm_tailcall_w2c_test_tailcaller_0(void **instance_ptr, void *tail_call_st struct wasm_multi_id w2c_test_infiniteloop(w2c_test* instance, u32 var_p0, f64 var_p1) { FUNC_PROLOGUE; + uint8_t* const wasm_rt_local_memory_base = NULL; + uint64_t wasm_rt_local_memory_size = 0; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_i0; f64 var_d1; var_i0 = var_p0; @@ -1092,6 +1138,10 @@ struct wasm_multi_id w2c_test_infiniteloop(w2c_test* instance, u32 var_p0, f64 v void wasm_tailcall_w2c_test_infiniteloop(void **instance_ptr, void *tail_call_stack, wasm_rt_tailcallee_t *next) { static_assert(sizeof(struct wasm_multi_id) <= 1024); w2c_test* instance = *instance_ptr; + uint8_t* const wasm_rt_local_memory_base = NULL; + uint64_t wasm_rt_local_memory_size = 0; + (void)wasm_rt_local_memory_base; + (void)wasm_rt_local_memory_size; u32 var_p0; f64 var_p1; { diff --git a/wasm2c/README.md b/wasm2c/README.md index b7ab3bca2e..cc3b44b63e 100644 --- a/wasm2c/README.md +++ b/wasm2c/README.md @@ -155,8 +155,8 @@ compiling a Wasm module with clang, running on x86_64 Linux, the macro `WASM_RT_ALLOW_SEGUE` is defined, and the flag `-mfsgsbase` is passed to clang. Segue is not used if -1. The Wasm module uses a more than a single unshared imported or exported - memory +1. The Wasm module does not use exactly one unshared, default-page, 32-bit + imported or exported memory. 2. The wasm2c code is compiled with GCC. Segue requires intrinsics for (rd|wr)gsbase, "address namespaces" for accessing pointers, and support for memcpy on pointers with custom "address namespaces". GCC does not support the