Skip to content

Commit

Permalink
Parametrize memory bounds checks on a per-memory basis
Browse files Browse the repository at this point in the history
  • Loading branch information
@keithw
keithw committed Nov 11, 2024
1 parent 176ec50 commit 5604e96
Show file tree
Hide file tree
Showing 17 changed files with 881 additions and 349 deletions.
19 changes: 18 additions & 1 deletion src/c-writer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1369,7 +1369,24 @@ static std::string GetMemoryTypeString(const Memory& memory) {
}

static std::string GetMemoryAPIString(const Memory& memory, std::string api) {
return memory.page_limits.is_shared ? (api + "_shared") : api;
std::string suffix;
if (memory.page_limits.is_shared) {
suffix += "_shared";
}

// Memory load and store routines can be optimized for default-page-size,
// 32-bit memories (by using hardware to bounds-check memory access).
// Append "_default32" to these function names to choose the (possibly) fast
// path.
//
// We don't need to do this for runtime routines; those can check the
// wasm_rt_memory_t structure.
if (api.substr(0, 8) != "wasm_rt_" &&
memory.page_size == WABT_DEFAULT_PAGE_SIZE &&
memory.page_limits.is_64 == false) {
suffix += "_default32";
}
return api + suffix;
}

void CWriter::WriteInitExpr(const ExprList& expr_list) {
Expand Down
136 changes: 73 additions & 63 deletions src/prebuilt/wasm2c_atomicops_source_declarations.cc
View file

Large diffs are not rendered by default.

46 changes: 26 additions & 20 deletions src/prebuilt/wasm2c_simd_source_declarations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,56 +15,62 @@ R"w2c_template(#endif
R"w2c_template(// TODO: equivalent constraint for ARM and other architectures
)w2c_template"
R"w2c_template(
#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \
#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \
)w2c_template"
R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \
R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t); \
R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \
)w2c_template"
R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \
R"w2c_template( SIMD_FORCE_READ(result); \
)w2c_template"
R"w2c_template( SIMD_FORCE_READ(result); \
R"w2c_template( return result; \
)w2c_template"
R"w2c_template( return result; \
R"w2c_template( } \
)w2c_template"
R"w2c_template( }
R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128);
)w2c_template"
R"w2c_template(
#define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \
)w2c_template"
R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \
R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t); \
R"w2c_template( v128 vec) { \
)w2c_template"
R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \
)w2c_template"
R"w2c_template( SIMD_FORCE_READ(result); \
)w2c_template"
R"w2c_template( return result; \
)w2c_template"
R"w2c_template( }
R"w2c_template( } \
)w2c_template"
R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128);
)w2c_template"
R"w2c_template(
#define DEFINE_SIMD_STORE(name, t) \
#define DEFINE_SIMD_STORE(name, t) \
)w2c_template"
R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \
R"w2c_template( v128 value) { \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t); \
R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \
)w2c_template"
R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \
R"w2c_template( } \
)w2c_template"
R"w2c_template( }
R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128);
)w2c_template"
R"w2c_template(
#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \
#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \
)w2c_template"
R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \
R"w2c_template( v128 value) { \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t); \
R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \
)w2c_template"
R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \
R"w2c_template( } \
)w2c_template"
R"w2c_template( }
R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128);
)w2c_template"
R"w2c_template(
// clang-format off
Expand Down
126 changes: 110 additions & 16 deletions src/prebuilt/wasm2c_source_declarations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,21 +218,40 @@ R"w2c_template(#define WASM_RT_CHECK_BASE(mem)
R"w2c_template(#endif
)w2c_template"
R"w2c_template(
#if WASM_RT_MEMCHECK_GUARD_PAGES
// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for
)w2c_template"
R"w2c_template(#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem);
R"w2c_template(// default-page-size, 32-bit memories. It may do nothing at all
)w2c_template"
R"w2c_template(// (if hardware bounds-checking is enabled via guard pages)
)w2c_template"
R"w2c_template(// or it may do a slightly faster RANGE_CHECK.
)w2c_template"
R"w2c_template(#if WASM_RT_MEMCHECK_GUARD_PAGES
)w2c_template"
R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem);
)w2c_template"
R"w2c_template(#else
)w2c_template"
R"w2c_template(#define MEMCHECK(mem, a, t) \
R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) \
)w2c_template"
R"w2c_template( WASM_RT_CHECK_BASE(mem); \
R"w2c_template( WASM_RT_CHECK_BASE(mem); \
)w2c_template"
R"w2c_template( RANGE_CHECK(mem, a, sizeof(t))
R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \
)w2c_template"
R"w2c_template( TRAP(OOB);
)w2c_template"
R"w2c_template(#endif
)w2c_template"
R"w2c_template(
// MEMCHECK_GENERAL can be used for any memory
)w2c_template"
R"w2c_template(#define MEMCHECK_GENERAL(mem, a, t) \
)w2c_template"
R"w2c_template( WASM_RT_CHECK_BASE(mem); \
)w2c_template"
R"w2c_template( RANGE_CHECK(mem, a, sizeof(t));
)w2c_template"
R"w2c_template(
#ifdef __GNUC__
)w2c_template"
R"w2c_template(#define FORCE_READ_INT(var) __asm__("" ::"r"(var));
Expand Down Expand Up @@ -302,38 +321,113 @@ R"w2c_template( load_data(MEM_ADDR(&m, o, s), i, s); \
R"w2c_template( } while (0)
)w2c_template"
R"w2c_template(
#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \
)w2c_template"
R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
)w2c_template"
R"w2c_template( u64 addr) { \
)w2c_template"
R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( ret_kw name##_unchecked(mem, addr); \
)w2c_template"
R"w2c_template( } \
)w2c_template"
R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \
)w2c_template"
R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( ret_kw name##_unchecked(mem, addr); \
)w2c_template"
R"w2c_template( }
)w2c_template"
R"w2c_template(
#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \
)w2c_template"
R"w2c_template( val_type1) \
)w2c_template"
R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
)w2c_template"
R"w2c_template( u64 addr, val_type1 val1) { \
)w2c_template"
R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \
)w2c_template"
R"w2c_template( } \
)w2c_template"
R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( val_type1 val1) { \
)w2c_template"
R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \
)w2c_template"
R"w2c_template( }
)w2c_template"
R"w2c_template(
#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \
)w2c_template"
R"w2c_template( val_type1, val_type2) \
)w2c_template"
R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
)w2c_template"
R"w2c_template( u64 addr, val_type1 val1, \
)w2c_template"
R"w2c_template( val_type2 val2) { \
)w2c_template"
R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \
R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t1); \
R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \
)w2c_template"
R"w2c_template( t1 result; \
R"w2c_template( } \
)w2c_template"
R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( sizeof(t1)); \
R"w2c_template( val_type1 val1, val_type2 val2) { \
)w2c_template"
R"w2c_template( force_read(result); \
R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \
)w2c_template"
R"w2c_template( return (t3)(t2)result; \
R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \
)w2c_template"
R"w2c_template( }
)w2c_template"
R"w2c_template(
#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
)w2c_template"
R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
)w2c_template"
R"w2c_template( t1 result; \
)w2c_template"
R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
)w2c_template"
R"w2c_template( sizeof(t1)); \
)w2c_template"
R"w2c_template( force_read(result); \
)w2c_template"
R"w2c_template( return (t3)(t2)result; \
)w2c_template"
R"w2c_template( } \
)w2c_template"
R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3)
)w2c_template"
R"w2c_template(
#define DEFINE_STORE(name, t1, t2) \
)w2c_template"
R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
)w2c_template"
R"w2c_template( MEMCHECK(mem, addr, t1); \
R"w2c_template( t2 value) { \
)w2c_template"
R"w2c_template( t1 wrapped = (t1)value; \
)w2c_template"
R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \
)w2c_template"
R"w2c_template( sizeof(t1)); \
)w2c_template"
R"w2c_template( }
R"w2c_template( } \
)w2c_template"
R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2)
)w2c_template"
R"w2c_template(
DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT)
Expand Down
81 changes: 65 additions & 16 deletions src/template/wasm2c.declarations.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,14 +117,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) {
#define WASM_RT_CHECK_BASE(mem)
#endif

// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for
// default-page-size, 32-bit memories. It may do nothing at all
// (if hardware bounds-checking is enabled via guard pages)
// or it may do a slightly faster RANGE_CHECK.
#if WASM_RT_MEMCHECK_GUARD_PAGES
#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem);
#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem);
#else
#define MEMCHECK(mem, a, t) \
WASM_RT_CHECK_BASE(mem); \
RANGE_CHECK(mem, a, sizeof(t))
#define MEMCHECK_DEFAULT32(mem, a, t) \
WASM_RT_CHECK_BASE(mem); \
if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \
TRAP(OOB);
#endif

// MEMCHECK_GENERAL can be used for any memory
#define MEMCHECK_GENERAL(mem, a, t) \
WASM_RT_CHECK_BASE(mem); \
RANGE_CHECK(mem, a, sizeof(t));

#ifdef __GNUC__
#define FORCE_READ_INT(var) __asm__("" ::"r"(var));
// Clang on Mips requires "f" constraints on floats
Expand Down Expand Up @@ -161,23 +171,62 @@ static inline void load_data(void* dest, const void* src, size_t n) {
load_data(MEM_ADDR(&m, o, s), i, s); \
} while (0)

#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \
MEMCHECK(mem, addr, t1); \
t1 result; \
wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
sizeof(t1)); \
force_read(result); \
return (t3)(t2)result; \
}
#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \
static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
u64 addr) { \
MEMCHECK_DEFAULT32(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr); \
} \
static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \
MEMCHECK_GENERAL(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr); \
}

#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \
val_type1) \
static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
u64 addr, val_type1 val1) { \
MEMCHECK_DEFAULT32(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr, val1); \
} \
static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
val_type1 val1) { \
MEMCHECK_GENERAL(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr, val1); \
}

#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \
val_type1, val_type2) \
static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \
u64 addr, val_type1 val1, \
val_type2 val2) { \
MEMCHECK_DEFAULT32(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr, val1, val2); \
} \
static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \
val_type1 val1, val_type2 val2) { \
MEMCHECK_GENERAL(mem, addr, mem_type); \
ret_kw name##_unchecked(mem, addr, val1, val2); \
}

#define DEFINE_LOAD(name, t1, t2, t3, force_read) \
static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \
t1 result; \
wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
sizeof(t1)); \
force_read(result); \
return (t3)(t2)result; \
} \
DEF_MEM_CHECKS0(name, _, t1, return, t3)

#define DEFINE_STORE(name, t1, t2) \
static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \
MEMCHECK(mem, addr, t1); \
static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \
t2 value) { \
t1 wrapped = (t1)value; \
wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \
sizeof(t1)); \
}
} \
DEF_MEM_CHECKS1(name, _, t1, , void, t2)

DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT)
DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT)
Expand Down
Loading

0 comments on commit 5604e96

Please sign in to comment.