diff --git a/bazel/seekdb_build_config.bzl b/bazel/seekdb_build_config.bzl index 29b0790b2..5aa24bec2 100644 --- a/bazel/seekdb_build_config.bzl +++ b/bazel/seekdb_build_config.bzl @@ -206,7 +206,10 @@ def seekdb_sanity_cxxopts(instrument): def seekdb_sanity_local_defines(): return select({ - _SEEKDB_SANITY_CONFIG: ["ENABLE_SANITY"], + _SEEKDB_SANITY_CONFIG: [ + "ENABLE_SANITY", + "OB_HAVE_BUNDLED_JEMALLOC=1", + ], "//conditions:default": [], }) diff --git a/bazel/third_party_headers_repo.bzl b/bazel/third_party_headers_repo.bzl index 219596b59..9ef88f47d 100644 --- a/bazel/third_party_headers_repo.bzl +++ b/bazel/third_party_headers_repo.bzl @@ -78,6 +78,12 @@ cc_library( includes = ["devtools/include"], ) +cc_library( + name = "jemalloc_headers", + hdrs = glob(["include/jemalloc/**/*." + extension for extension in HEADER_EXTENSIONS], allow_empty = True), + includes = ["include"], +) + filegroup( name = "sanity_pass", srcs = ["devtools/lib64/libsanitypass.so"], diff --git a/src/oblib/BUILD.bazel b/src/oblib/BUILD.bazel index 3810c0fd5..2a4a2916b 100644 --- a/src/oblib/BUILD.bazel +++ b/src/oblib/BUILD.bazel @@ -203,6 +203,7 @@ seekdb_semantic_unity_cc_library( implementation_deps = [ ":_oblib_implementation_headers", ":oblib_foundation", + "@seekdb_3rd_headers//:jemalloc_headers", ], copts = SEEKDB_OBLIB_COPTS, local_defines = SEEKDB_OBLIB_LOCAL_DEFINES, @@ -420,13 +421,7 @@ cc_library( name = "ob_malloc_objects", implementation_deps = [ ":_ob_malloc_impl", - ] + select({ - "//bazel:sanity_enabled": [ - "//bazel:sanity_abort_runtime", - "@seekdb_3rd_headers//:sanity_runtime", - ], - "//conditions:default": [], - }), + ], tags = ["manual"], visibility = ["//src/observer:__pkg__"], ) diff --git a/src/oblib/lib/allocator/ob_jemalloc.h b/src/oblib/lib/allocator/ob_jemalloc.h index e9b6de7d2..ac385065b 100644 --- a/src/oblib/lib/allocator/ob_jemalloc.h +++ b/src/oblib/lib/allocator/ob_jemalloc.h @@ -38,10 +38,25 @@ namespace oceanbase namespace common { +#if defined(ENABLE_SANITY) && defined(OB_HAVE_BUNDLED_JEMALLOC) +void *jemalloc_sanity_malloc(size_t size) noexcept; +void jemalloc_sanity_free(void *ptr) noexcept; +void *jemalloc_sanity_realloc(void *ptr, size_t size) noexcept; +void *jemalloc_sanity_memalign(size_t alignment, size_t size) noexcept; +size_t jemalloc_sanity_usable_size(void *ptr) noexcept; +bool jemalloc_sanity_enable_background_threads() noexcept; +void jemalloc_sanity_poison(const void *ptr, size_t size) noexcept; +void jemalloc_sanity_unpoison(const void *ptr, size_t size) noexcept; +#endif + inline void *jemalloc_malloc(const size_t size) { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + return jemalloc_sanity_malloc(size); +#else return je_malloc(size); +#endif #else (void)size; return nullptr; @@ -51,7 +66,11 @@ inline void *jemalloc_malloc(const size_t size) inline void jemalloc_free(void *ptr) { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + jemalloc_sanity_free(ptr); +#else je_free(ptr); +#endif #else (void)ptr; #endif @@ -60,7 +79,11 @@ inline void jemalloc_free(void *ptr) inline void *jemalloc_realloc(void *ptr, const size_t size) { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + return jemalloc_sanity_realloc(ptr, size); +#else return je_realloc(ptr, size); +#endif #else (void)ptr; (void)size; @@ -71,12 +94,16 @@ inline void *jemalloc_realloc(void *ptr, const size_t size) inline void *jemalloc_memalign(const size_t alignment, const size_t size) { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + return jemalloc_sanity_memalign(alignment, size); +#else #if defined(__APPLE__) void *ptr = nullptr; return 0 == je_posix_memalign(&ptr, alignment, size) ? ptr : nullptr; #else return je_memalign(alignment, size); #endif +#endif #else (void)alignment; (void)size; @@ -87,7 +114,11 @@ inline void *jemalloc_memalign(const size_t alignment, const size_t size) inline size_t jemalloc_usable_size(void *ptr) { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + return nullptr == ptr ? 0 : jemalloc_sanity_usable_size(ptr); +#else return nullptr == ptr ? 0 : je_malloc_usable_size(ptr); +#endif #else (void)ptr; return 0; @@ -97,9 +128,13 @@ inline size_t jemalloc_usable_size(void *ptr) inline bool jemalloc_enable_background_threads() { #if defined(OB_HAVE_BUNDLED_JEMALLOC) +#if defined(ENABLE_SANITY) + return jemalloc_sanity_enable_background_threads(); +#else bool enabled = true; return 0 == je_mallctl("background_thread", nullptr, nullptr, &enabled, sizeof(enabled)); +#endif #else return true; #endif diff --git a/src/oblib/lib/allocator/ob_jemalloc_sanity.cpp b/src/oblib/lib/allocator/ob_jemalloc_sanity.cpp new file mode 100644 index 000000000..bcbe76d4f --- /dev/null +++ b/src/oblib/lib/allocator/ob_jemalloc_sanity.cpp @@ -0,0 +1,479 @@ +/* + * Copyright (c) 2026 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lib/allocator/ob_jemalloc.h" + +#if defined(ENABLE_SANITY) && defined(OB_HAVE_BUNDLED_JEMALLOC) && \ + defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// The packaged libsanity runtime interposes libc through dlsym and can recurse +// into malloc before seekdb has entered main(). The compiler pass only needs +// these globals and memory_sanity_abort(); libc checks are supplied by +// ob_sanity_libc_wrap.cpp without dlsym. +int64_t sanity_min_addr = 0; +int64_t sanity_max_addr = 0; + +namespace oceanbase { +namespace common { +namespace { + +// The former OBMalloc Sanity integration searched the same upper bounds. Its +// maximum candidate was [0x0c0000000000, 0x600000000000): about 84 TiB of +// application address space plus 10.5 TiB of shadow; occupied mappings made it +// retreat in 128 GiB steps. Preserve that policy instead of the 64 GiB limit +// used by the first jemalloc proof of concept. +constexpr uintptr_t HEAP_MAX_CANDIDATES[] = { + 0x600000000000ULL, + 0x500000000000ULL, + 0x400000000000ULL, +}; +constexpr size_t ADDRESS_SEARCH_STEP = 128ULL << 30; +constexpr size_t BOOTSTRAP_GAP = 2ULL << 20; +constexpr size_t REDZONE_SIZE = 16; + +struct AllocationHeader { + void *base_; + size_t requested_; +}; + +struct ReservedRegion { + uintptr_t begin_ = 0; + uintptr_t end_ = 0; + + size_t size() const { return end_ - begin_; } + uintptr_t shadow_begin() const { return begin_ >> 3; } + uintptr_t shadow_end() const { return end_ >> 3; } + size_t shadow_size() const { return shadow_end() - shadow_begin(); } + bool valid() const { return begin_ > 0 && end_ > begin_; } +}; + +std::atomic heap_begin{0}; +std::atomic heap_end{0}; +std::atomic next_extent_addr{0}; +std::atomic init_state{0}; +__thread bool initializing_arena = false; +unsigned sanity_arena = 0; + +uintptr_t align_up(uintptr_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +bool reserve_exact(uintptr_t address, size_t size) { + constexpr int BASE_FLAGS = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + void *result = MAP_FAILED; +#if defined(MAP_FIXED_NOREPLACE) + result = mmap(reinterpret_cast(address), size, PROT_NONE, + BASE_FLAGS | MAP_FIXED_NOREPLACE, -1, 0); + if (MAP_FAILED == result && EINVAL == errno) { + // Linux before 4.17 does not understand MAP_FIXED_NOREPLACE. A plain + // address hint is still safe: reject and unmap any non-exact result. + result = mmap(reinterpret_cast(address), size, PROT_NONE, + BASE_FLAGS, -1, 0); + } +#else + result = mmap(reinterpret_cast(address), size, PROT_NONE, BASE_FLAGS, + -1, 0); +#endif + if (result != reinterpret_cast(address)) { + if (MAP_FAILED != result) { + static_cast(munmap(result, size)); + } + return false; + } + return true; +} + +void release_region(const ReservedRegion ®ion) { + if (region.valid()) { + static_cast( + munmap(reinterpret_cast(region.begin_), region.size())); + static_cast(munmap(reinterpret_cast(region.shadow_begin()), + region.shadow_size())); + } +} + +bool reserve_region(const ReservedRegion ®ion) { + if (!region.valid() || !reserve_exact(region.begin_, region.size())) { + return false; + } + if (!reserve_exact(region.shadow_begin(), region.shadow_size())) { + static_cast( + munmap(reinterpret_cast(region.begin_), region.size())); + return false; + } + return true; +} + +ReservedRegion probe_largest_region() { + ReservedRegion best; + for (const uintptr_t candidate_end : HEAP_MAX_CANDIDATES) { + for (uintptr_t candidate_begin = candidate_end >> 3; + candidate_begin < candidate_end; + candidate_begin += ADDRESS_SEARCH_STEP) { + const ReservedRegion candidate{candidate_begin, candidate_end}; + if (reserve_region(candidate)) { + release_region(candidate); + if (candidate.size() > best.size()) { + best = candidate; + } + break; + } + } + } + return best; +} + +bool reserve_sanity_region(ReservedRegion &selected) { + // Retry the selection if another pre-main mapping appears between the probe + // and the final reservation. + for (int attempt = 0; attempt < 3; ++attempt) { + selected = probe_largest_region(); + if (selected.valid() && reserve_region(selected)) { + static_cast(madvise(reinterpret_cast(selected.begin_), + selected.size(), MADV_DONTDUMP)); + static_cast( + madvise(reinterpret_cast(selected.shadow_begin()), + selected.shadow_size(), MADV_DONTDUMP)); + return true; + } + } + return false; +} + +bool make_shadow_accessible(uintptr_t address, size_t size) { + const size_t page_size = static_cast(sysconf(_SC_PAGESIZE)); + const uintptr_t shadow_begin = address >> 3; + const uintptr_t shadow_end = (address + size + 7) >> 3; + const uintptr_t page_begin = shadow_begin & ~(page_size - 1); + const uintptr_t page_end = align_up(shadow_end, page_size); + return 0 == mprotect(reinterpret_cast(page_begin), + page_end - page_begin, PROT_READ | PROT_WRITE); +} + +void set_shadow(void *ptr, size_t size, uint8_t value) { + volatile uint8_t *shadow = reinterpret_cast( + reinterpret_cast(ptr) >> 3); + const size_t shadow_size = (size + 7) >> 3; + for (size_t i = 0; i < shadow_size; ++i) { + shadow[i] = value; + } +} + +void unpoison_user_memory(void *ptr, size_t size) { + // Returned pointers are max_align_t-aligned and therefore start on a shadow + // boundary. Write shadow directly to avoid sanity_unpoison()'s first-use + // dlsym allocation while jemalloc may hold an arena lock. + volatile uint8_t *shadow = reinterpret_cast( + reinterpret_cast(ptr) >> 3); + const size_t full_blocks = size >> 3; + for (size_t i = 0; i < full_blocks; ++i) { + shadow[i] = 0; + } + if (0 != (size & 7)) { + shadow[full_blocks] = static_cast(size & 7); + } +} + +void *extent_alloc(extent_hooks_t *, void *new_addr, size_t size, + size_t alignment, bool *zero, bool *commit, unsigned) { + if (nullptr != new_addr) { + return nullptr; + } + uintptr_t current = next_extent_addr.load(std::memory_order_relaxed); + const uintptr_t end = heap_end.load(std::memory_order_relaxed); + uintptr_t allocated = 0; + do { + allocated = + align_up(current, std::max(alignment, static_cast(4096))); + if (size > end || allocated >= end || size > end - allocated) { + return nullptr; + } + } while (!next_extent_addr.compare_exchange_weak(current, allocated + size, + std::memory_order_relaxed)); + + void *mapped = + mmap(reinterpret_cast(allocated), size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (mapped != reinterpret_cast(allocated)) { + return nullptr; + } + if (!make_shadow_accessible(allocated, size)) { + static_cast( + mmap(reinterpret_cast(allocated), size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED | MAP_NORESERVE, -1, 0)); + return nullptr; + } + set_shadow(mapped, size, 0xF0); + *zero = true; + *commit = true; + return mapped; +} + +bool extent_dalloc(extent_hooks_t *, void *, size_t, bool, unsigned) { + return true; +} +void extent_destroy(extent_hooks_t *, void *, size_t, bool, unsigned) {} +bool extent_commit(extent_hooks_t *, void *, size_t, size_t, size_t, unsigned) { + return false; +} +bool extent_decommit(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned) { + return true; +} +bool extent_purge(extent_hooks_t *, void *addr, size_t, size_t offset, + size_t length, unsigned) { + return 0 != + madvise(static_cast(addr) + offset, length, MADV_DONTNEED); +} +bool extent_split(extent_hooks_t *, void *, size_t, size_t, size_t, bool, + unsigned) { + return false; +} +bool extent_merge(extent_hooks_t *, void *, size_t, void *, size_t, bool, + unsigned) { + return false; +} + +extent_hooks_t hooks = {extent_alloc, extent_dalloc, extent_destroy, + extent_commit, extent_decommit, extent_purge, + extent_purge, extent_split, extent_merge}; + +bool initialize_arena() { + ReservedRegion selected; + if (!reserve_sanity_region(selected)) { + return false; + } + heap_begin.store(selected.begin_, std::memory_order_relaxed); + heap_end.store(selected.end_, std::memory_order_relaxed); + next_extent_addr.store(selected.begin_ + BOOTSTRAP_GAP, + std::memory_order_relaxed); + sanity_min_addr = static_cast(selected.begin_); + sanity_max_addr = static_cast(selected.end_); + extent_hooks_t *hook_ptr = &hooks; + size_t arena_size = sizeof(sanity_arena); + SanityDisableCheckRangeGuard guard; + if (0 != je_mallctl("arenas.create", &sanity_arena, &arena_size, &hook_ptr, + sizeof(hook_ptr))) { + release_region(selected); + sanity_min_addr = 0; + sanity_max_addr = 0; + heap_begin.store(0, std::memory_order_relaxed); + heap_end.store(0, std::memory_order_relaxed); + return false; + } + return true; +} + +bool ensure_initialized() { + int state = init_state.load(std::memory_order_acquire); + if (2 == state) { + return true; + } + int expected = 0; + if (init_state.compare_exchange_strong(expected, 1, + std::memory_order_acq_rel)) { + initializing_arena = true; + const bool success = initialize_arena(); + initializing_arena = false; + init_state.store(success ? 2 : -1, std::memory_order_release); + return success; + } + while (1 == (state = init_state.load(std::memory_order_acquire))) { + syscall(SYS_sched_yield); + } + return 2 == state; +} + +__attribute__((constructor(200))) void initialize_jemalloc_sanity_arena() { + if (!ensure_initialized()) { + static constexpr char MESSAGE[] = + "seekdb: failed to initialize jemalloc sanity arena\n"; + syscall(SYS_write, STDERR_FILENO, MESSAGE, sizeof(MESSAGE) - 1); + syscall(SYS_exit_group, 127); + __builtin_unreachable(); + } +} + +int flags() { return MALLOCX_ARENA(sanity_arena) | MALLOCX_TCACHE_NONE; } + +AllocationHeader *header_from_user(void *ptr) { + return reinterpret_cast(ptr) - 1; +} + +void *allocate_aligned(size_t alignment, size_t size) { + // arenas.create may allocate through the process-wide malloc symbol. Those + // bootstrap allocations must use jemalloc's default arena or initialization + // would recursively wait for itself. + if (initializing_arena) { + SanityDisableCheckRangeGuard guard; + return alignment <= alignof(std::max_align_t) + ? je_malloc(size) + : je_memalign(alignment, size); + } + if (!ensure_initialized()) { + return nullptr; + } + alignment = std::max(alignment, alignof(std::max_align_t)); + if (0 != (alignment & (alignment - 1))) { + return nullptr; + } + constexpr size_t FIXED_OVERHEAD = sizeof(AllocationHeader) + REDZONE_SIZE; + if (alignment > SIZE_MAX - FIXED_OVERHEAD || + size > SIZE_MAX - FIXED_OVERHEAD - (alignment - 1)) { + return nullptr; + } + const size_t total = size + FIXED_OVERHEAD + alignment - 1; + void *base = nullptr; + { + SanityDisableCheckRangeGuard guard; + base = je_mallocx(total, flags()); + } + if (nullptr == base) { + return nullptr; + } + const uintptr_t user_addr = align_up( + reinterpret_cast(base) + sizeof(AllocationHeader), alignment); + void *user = reinterpret_cast(user_addr); + AllocationHeader *header = header_from_user(user); + header->base_ = base; + header->requested_ = size; + unpoison_user_memory(user, size); + return user; +} + +} // namespace + +void *jemalloc_sanity_malloc(size_t size) noexcept { + return allocate_aligned(alignof(std::max_align_t), size); +} + +void jemalloc_sanity_free(void *ptr) noexcept { + if (nullptr != ptr) { + if (!sanity_addr_in_range(ptr, 0)) { + SanityDisableCheckRangeGuard guard; + je_free(ptr); + return; + } + AllocationHeader *header = header_from_user(ptr); + void *base = header->base_; + size_t usable = 0; + { + SanityDisableCheckRangeGuard guard; + usable = je_sallocx(base, 0); + } + set_shadow(base, usable, 0xF0); + { + SanityDisableCheckRangeGuard guard; + je_dallocx(base, flags()); + } + } +} + +void *jemalloc_sanity_realloc(void *ptr, size_t size) noexcept { + if (nullptr == ptr) { + return jemalloc_sanity_malloc(size); + } + if (0 == size) { + jemalloc_sanity_free(ptr); + return nullptr; + } + if (!sanity_addr_in_range(ptr, 0)) { + SanityDisableCheckRangeGuard guard; + return je_realloc(ptr, size); + } + AllocationHeader *old_header = header_from_user(ptr); + const size_t old_size = old_header->requested_; + void *new_ptr = jemalloc_sanity_malloc(size); + if (nullptr != new_ptr) { + std::memcpy(new_ptr, ptr, std::min(old_size, size)); + jemalloc_sanity_free(ptr); + } + return new_ptr; +} + +void *jemalloc_sanity_memalign(size_t alignment, size_t size) noexcept { + return allocate_aligned(alignment, size); +} + +size_t jemalloc_sanity_usable_size(void *ptr) noexcept { + if (sanity_addr_in_range(ptr, 0)) { + return header_from_user(ptr)->requested_; + } + SanityDisableCheckRangeGuard guard; + return je_malloc_usable_size(ptr); +} + +bool jemalloc_sanity_enable_background_threads() noexcept { + // A jemalloc background thread has no guarded allocator-call boundary and + // legitimately accesses metadata that remains poisoned to application code. + return true; +} + +void jemalloc_sanity_poison(const void *ptr, size_t size) noexcept { + const uintptr_t begin = reinterpret_cast(ptr); + const uintptr_t range_begin = heap_begin.load(std::memory_order_relaxed); + const uintptr_t range_end = heap_end.load(std::memory_order_relaxed); + if (nullptr != ptr && begin >= range_begin && begin < range_end && + size <= range_end - begin) { + const uintptr_t aligned_begin = align_up(begin, 8); + if (aligned_begin - begin < size) { + const size_t aligned_size = size - (aligned_begin - begin); + set_shadow(reinterpret_cast(aligned_begin), aligned_size, 0xF0); + } + } +} + +void jemalloc_sanity_unpoison(const void *ptr, size_t size) noexcept { + const uintptr_t begin = reinterpret_cast(ptr); + const uintptr_t range_begin = heap_begin.load(std::memory_order_relaxed); + const uintptr_t range_end = heap_end.load(std::memory_order_relaxed); + if (nullptr != ptr && begin >= range_begin && begin < range_end && + size <= range_end - begin) { + const uintptr_t aligned_begin = align_up(begin, 8); + if (aligned_begin - begin < size) { + const size_t aligned_size = size - (aligned_begin - begin); + unpoison_user_memory(reinterpret_cast(aligned_begin), aligned_size); + } + } +} + +} // namespace common +} // namespace oceanbase + +extern "C" void memory_sanity_abort() { + static constexpr char MESSAGE[] = "seekdb: memory sanity check failed\n"; + SanityDisableCheckRangeGuard guard; + syscall(SYS_write, STDERR_FILENO, MESSAGE, sizeof(MESSAGE) - 1); + void *frames[32]; + const int frame_count = backtrace(frames, sizeof(frames) / sizeof(frames[0])); + backtrace_symbols_fd(frames, frame_count, STDERR_FILENO); + __builtin_trap(); +} + +#endif diff --git a/src/oblib/lib/allocator/ob_malloc.cpp b/src/oblib/lib/allocator/ob_malloc.cpp index b415208e5..eb0cd45ed 100644 --- a/src/oblib/lib/allocator/ob_malloc.cpp +++ b/src/oblib/lib/allocator/ob_malloc.cpp @@ -32,8 +32,13 @@ #if defined(OB_HAVE_BUNDLED_JEMALLOC) extern "C" { +#if defined(ENABLE_SANITY) +const char *je_malloc_conf = + "background_thread:false,dirty_decay_ms:1000,muzzy_decay_ms:0"; +#else const char *je_malloc_conf = "background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:0"; +#endif } #endif diff --git a/src/oblib/lib/allocator/ob_sanity_libc_wrap.cpp b/src/oblib/lib/allocator/ob_sanity_libc_wrap.cpp new file mode 100644 index 000000000..125c50a9a --- /dev/null +++ b/src/oblib/lib/allocator/ob_sanity_libc_wrap.cpp @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2026 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(ENABLE_SANITY) && defined(__linux__) + +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +void check_sanity_range(const void *ptr, size_t size) { + if (!SanityDisableCheckRangeGuard::tl_check() || nullptr == ptr || + 0 == size) { + return; + } + const uintptr_t begin = reinterpret_cast(ptr); + if (begin < static_cast(sanity_min_addr) || + begin >= static_cast(sanity_max_addr)) { + return; + } + if (size > static_cast(sanity_max_addr) - begin) { + memory_sanity_abort(); + } + + uintptr_t current = begin; + const uintptr_t end = begin + size; + while (current < end) { + const uint8_t shadow = *reinterpret_cast(current >> 3); + const size_t offset = current & 7; + const size_t bytes_in_block = static_cast( + ((current | static_cast(7)) + 1) - current); + const size_t checked = bytes_in_block < end - current + ? bytes_in_block + : static_cast(end - current); + const size_t accessible = 0 == shadow ? 8 : (shadow <= 7 ? shadow : 0); + if (offset + checked > accessible) { + memory_sanity_abort(); + } + current += checked; + } +} + +size_t checked_string_size(const char *str, size_t limit) { + size_t size = 0; + while (size < limit) { + check_sanity_range(str + size, 1); + if ('\0' == str[size++]) { + break; + } + } + return size; +} + +} // namespace + +extern "C" { + +void *__real_memcpy(void *, const void *, size_t); +void *__real_memmove(void *, const void *, size_t); +void *__real_memset(void *, int, size_t); +int __real_memcmp(const void *, const void *, size_t); +char *__real_strcpy(char *, const char *); +char *__real_strncpy(char *, const char *, size_t); +int __real_strcasecmp(const char *, const char *); +int __real_strncasecmp(const char *, const char *, size_t); +int __real_vsprintf(char *, const char *, va_list); +int __real_vsnprintf(char *, size_t, const char *, va_list); + +void *__wrap_memcpy(void *dst, const void *src, size_t size) { + check_sanity_range(src, size); + check_sanity_range(dst, size); + return __real_memcpy(dst, src, size); +} + +void *__wrap_memmove(void *dst, const void *src, size_t size) { + check_sanity_range(src, size); + check_sanity_range(dst, size); + return __real_memmove(dst, src, size); +} + +void *__wrap_memset(void *dst, int value, size_t size) { + check_sanity_range(dst, size); + return __real_memset(dst, value, size); +} + +void __wrap_bzero(void *dst, size_t size) { + check_sanity_range(dst, size); + static_cast(__real_memset(dst, 0, size)); +} + +int __wrap_memcmp(const void *lhs, const void *rhs, size_t size) { + check_sanity_range(lhs, size); + check_sanity_range(rhs, size); + return __real_memcmp(lhs, rhs, size); +} + +size_t __wrap_strlen(const char *str) { + return checked_string_size(str, SIZE_MAX) - 1; +} + +size_t __wrap_strnlen(const char *str, size_t max_size) { + const size_t checked = checked_string_size(str, max_size); + return checked > 0 && '\0' == str[checked - 1] ? checked - 1 : checked; +} + +char *__wrap_strcpy(char *dst, const char *src) { + const size_t size = checked_string_size(src, SIZE_MAX); + check_sanity_range(dst, size); + return __real_strcpy(dst, src); +} + +char *__wrap_strncpy(char *dst, const char *src, size_t size) { + static_cast(checked_string_size(src, size)); + check_sanity_range(dst, size); + return __real_strncpy(dst, src, size); +} + +int __wrap_strcmp(const char *lhs, const char *rhs) { + for (size_t i = 0;; ++i) { + check_sanity_range(lhs + i, 1); + check_sanity_range(rhs + i, 1); + const unsigned char left = static_cast(lhs[i]); + const unsigned char right = static_cast(rhs[i]); + if (left != right || '\0' == left) { + return static_cast(left) - static_cast(right); + } + } +} + +int __wrap_strncmp(const char *lhs, const char *rhs, size_t size) { + for (size_t i = 0; i < size; ++i) { + check_sanity_range(lhs + i, 1); + check_sanity_range(rhs + i, 1); + const unsigned char left = static_cast(lhs[i]); + const unsigned char right = static_cast(rhs[i]); + if (left != right || '\0' == left) { + return static_cast(left) - static_cast(right); + } + } + return 0; +} + +int __wrap_strcasecmp(const char *lhs, const char *rhs) { + static_cast(checked_string_size(lhs, SIZE_MAX)); + static_cast(checked_string_size(rhs, SIZE_MAX)); + return __real_strcasecmp(lhs, rhs); +} + +int __wrap_strncasecmp(const char *lhs, const char *rhs, size_t size) { + static_cast(checked_string_size(lhs, size)); + static_cast(checked_string_size(rhs, size)); + return __real_strncasecmp(lhs, rhs, size); +} + +int __wrap_vsnprintf(char *dst, size_t size, const char *format, va_list args) { + static_cast(checked_string_size(format, SIZE_MAX)); + check_sanity_range(dst, size); + return __real_vsnprintf(dst, size, format, args); +} + +int __wrap_snprintf(char *dst, size_t size, const char *format, ...) { + va_list args; + va_start(args, format); + const int result = __wrap_vsnprintf(dst, size, format, args); + va_end(args); + return result; +} + +int __wrap_vsprintf(char *dst, const char *format, va_list args) { + static_cast(checked_string_size(format, SIZE_MAX)); + va_list size_args; + va_copy(size_args, args); + const int output_size = __real_vsnprintf(nullptr, 0, format, size_args); + va_end(size_args); + if (output_size >= 0) { + check_sanity_range(dst, static_cast(output_size) + 1); + } + return __real_vsprintf(dst, format, args); +} + +int __wrap_sprintf(char *dst, const char *format, ...) { + va_list args; + va_start(args, format); + const int result = __wrap_vsprintf(dst, format, args); + va_end(args); + return result; +} + +} // extern "C" + +#endif diff --git a/src/oblib/lib/allocator/page_arena.h b/src/oblib/lib/allocator/page_arena.h index 40068e443..c008cc91c 100644 --- a/src/oblib/lib/allocator/page_arena.h +++ b/src/oblib/lib/allocator/page_arena.h @@ -272,8 +272,58 @@ class PageArena int64_t total_; // total number of bytes occupied by pages PageAllocatorT page_allocator_; TracerContext *tc_; + bool enable_sanity_; private: // helpers + static bool sanity_enabled(const bool enabled) + { +#if defined(ENABLE_SANITY) && defined(OB_HAVE_BUNDLED_JEMALLOC) && defined(__linux__) + return enabled; +#else + UNUSED(enabled); + return false; +#endif + } + + static void sanity_poison(const void *ptr, const int64_t size) + { +#if defined(ENABLE_SANITY) && defined(OB_HAVE_BUNDLED_JEMALLOC) && defined(__linux__) + if (size > 0) { + jemalloc_sanity_poison(ptr, static_cast(size)); + } +#else + UNUSED(ptr); + UNUSED(size); +#endif + } + + static void sanity_unpoison(const void *ptr, const int64_t size) + { +#if defined(ENABLE_SANITY) && defined(OB_HAVE_BUNDLED_JEMALLOC) && defined(__linux__) + if (size > 0) { + jemalloc_sanity_unpoison(ptr, static_cast(size)); + } +#else + UNUSED(ptr); + UNUSED(size); +#endif + } + + static bool sanity_raw_size(const int64_t size, int64_t &raw_size) + { + bool valid = size > 0 && size <= INT64_MAX - 15; + if (valid) { + raw_size = lib::align_up2(size, 8) + 8; + } + return valid; + } + + const char *sanity_page_end(const Page *page) const + { + const char *normal_page_end = reinterpret_cast(page) + page_size_; + return page->page_end_ < normal_page_end ? normal_page_end : page->page_end_; + } + Page *insert_head(Page *page) { if (OB_ISNULL(page)) { @@ -303,6 +353,9 @@ class PageArena if (NULL != ptr) { page = new(ptr) Page((char *)ptr + sz); + if (sanity_enabled(enable_sanity_)) { + sanity_poison(page->buf_, page->page_end_ - page->buf_); + } total_ += sz; ++pages_; } else { @@ -315,6 +368,9 @@ class PageArena } void free_page(Page *page) { + if (sanity_enabled(enable_sanity_)) { + sanity_unpoison(page->buf_, sanity_page_end(page) - page->buf_); + } page_allocator_.free(page); } @@ -428,6 +484,7 @@ class PageArena page_size_ = rhs.page_size_; page_limit_ = rhs.page_limit_; page_allocator_ = rhs.page_allocator_; + enable_sanity_ = rhs.enable_sanity_; } return *this; @@ -436,10 +493,12 @@ class PageArena public: // API /** constructor */ PageArena(const int64_t page_size, - const PageAllocatorT &alloc) + const PageAllocatorT &alloc, + const bool enable_sanity = false) : cur_page_(NULL), header_(NULL), tailer_(NULL), page_limit_(0), page_size_(page_size), - pages_(0), used_(0), total_(0), page_allocator_(alloc), tc_(nullptr) + pages_(0), used_(0), total_(0), page_allocator_(alloc), tc_(nullptr), + enable_sanity_(enable_sanity) { if (page_size < (int64_t)sizeof(Page)) { _OB_LOG_RET(ERROR, OB_ERROR, "invalid page size(page_size=%ld, page=%ld)", page_size, @@ -447,7 +506,7 @@ class PageArena } } PageArena(const int64_t page_size) - : PageArena(page_size, PageAllocatorT()) {} + : PageArena(page_size, PageAllocatorT(), true) {} PageArena() : PageArena(DEFAULT_PAGE_SIZE) {} virtual ~PageArena() { free(); } @@ -540,7 +599,20 @@ class PageArena } CharT *alloc(const int64_t sz) { - return _alloc(sz); + CharT *ret = NULL; + if (!sanity_enabled(enable_sanity_)) { + ret = _alloc(sz); + } else { + int64_t raw_size = 0; + if (sanity_raw_size(sz, raw_size)) { + ret = _alloc(raw_size); + if (NULL != ret) { + sanity_unpoison(ret, sz); + sanity_poison(ret + sz, 8); + } + } + } + return ret; } CharT *alloc(const int64_t sz, const lib::ObMemAttr &attr) { @@ -605,14 +677,27 @@ class PageArena CharT *alloc_aligned(const int64_t sz, const int64_t alignment = 16) { - return _alloc_aligned(sz, alignment); + CharT *ret = NULL; + if (!sanity_enabled(enable_sanity_)) { + ret = _alloc_aligned(sz, alignment); + } else { + int64_t raw_size = 0; + if (sanity_raw_size(sz, raw_size)) { + ret = _alloc_aligned(raw_size, alignment); + if (NULL != ret) { + sanity_unpoison(ret, sz); + sanity_poison(ret + sz, 8); + } + } + } + return ret; } /** * allocate from the end of the page. * - allow better packing/space saving for certain scenarios */ - CharT *alloc_down(const int64_t sz) + CharT *_alloc_down(const int64_t sz) { // common case CharT *ret = NULL; @@ -645,6 +730,24 @@ class PageArena return ret; } + CharT *alloc_down(const int64_t sz) + { + CharT *ret = NULL; + if (!sanity_enabled(enable_sanity_)) { + ret = _alloc_down(sz); + } else { + int64_t raw_size = 0; + if (sanity_raw_size(sz, raw_size)) { + ret = _alloc_down(raw_size); + if (NULL != ret) { + sanity_unpoison(ret, sz); + sanity_poison(ret + sz, 8); + } + } + } + return ret; + } + /** realloc for newsz bytes */ CharT *realloc(CharT *p, const int64_t oldsz, const int64_t newsz) { @@ -653,7 +756,7 @@ class PageArena } else { ret = p; // if we're the last one on the current page with enough space - if (p + oldsz == cur_page_->alloc_end_ + if (!sanity_enabled(enable_sanity_) && p + oldsz == cur_page_->alloc_end_ && p + newsz < cur_page_->page_end_) { cur_page_->alloc_end_ = (char *)p + newsz; ret = p; @@ -702,7 +805,7 @@ class PageArena * * @return nullptr when failed */ - CharT *alloc_aligned_bf(const int64_t sz, const int64_t alignment) + CharT *_alloc_aligned_bf(const int64_t sz, const int64_t alignment) { assert(alignment >=0 && alignment <= UINT32_MAX); assert(ob_is_power_of_two(static_cast(alignment))); @@ -756,6 +859,24 @@ class PageArena return ret; } + CharT *alloc_aligned_bf(const int64_t sz, const int64_t alignment) + { + CharT *ret = NULL; + if (!sanity_enabled(enable_sanity_)) { + ret = _alloc_aligned_bf(sz, alignment); + } else { + int64_t raw_size = 0; + if (sanity_raw_size(sz, raw_size)) { + ret = _alloc_aligned_bf(raw_size, alignment); + if (NULL != ret) { + sanity_unpoison(ret, sz); + sanity_poison(ret + sz, 8); + } + } + } + return ret; + } + /** free the whole arena */ void free() @@ -796,6 +917,10 @@ class PageArena } page = NULL; } + if (NULL != remain_page && sanity_enabled(enable_sanity_)) { + sanity_poison(remain_page->buf_, + sanity_page_end(remain_page) - remain_page->buf_); + } header_ = cur_page_ = remain_page; if (NULL == cur_page_) { page_allocator_.freed(total_); @@ -855,7 +980,16 @@ class PageArena // CANNOT use anymore. cur_page_ = header_; if (NULL == header_) { tailer_ = NULL; } + if (sanity_enabled(enable_sanity_)) { + Page *remain_page = header_; + while (NULL != remain_page) { + sanity_poison(remain_page->buf_, + sanity_page_end(remain_page) - remain_page->buf_); + remain_page = remain_page->next_page_; + } + } used_ = 0; + tc_ = nullptr; } //[[deprecated("Arena is not allowed to call free(ptr), use free() instead")]] @@ -903,6 +1037,15 @@ class PageArena { bool bret = true; if (nullptr != tc_) { + Page *traced_page = tc_->cur_page_.next_page_; + char *traced_alloc_end = tc_->cur_page_.alloc_end_; + const char *traced_page_end = tc_->cur_page_.page_end_; + if (sanity_enabled(enable_sanity_) && NULL != traced_page) { + sanity_poison(traced_alloc_end, + traced_page->alloc_end_ - traced_alloc_end); + sanity_poison(traced_page->page_end_, + traced_page_end - traced_page->page_end_); + } // Free large pages from current header to header of trace pointer. // Free normal pages from trace pointer page to current page. // Restore current page and statistics information. @@ -926,6 +1069,8 @@ class PageArena free_page(page); page = next_page; } + cur_page_->alloc_end_ = traced_alloc_end; + cur_page_->page_end_ = traced_page_end; // 3. restore statistics pages_ = tc_->pages_; @@ -943,6 +1088,13 @@ class PageArena void fast_reuse() { + if (sanity_enabled(enable_sanity_)) { + Page *page = header_; + while (NULL != page) { + sanity_poison(page->buf_, sanity_page_end(page) - page->buf_); + page = page->next_page_; + } + } used_ = 0; cur_page_ = header_; if (NULL != cur_page_) { @@ -987,12 +1139,14 @@ class ObArenaAllocator final : public ObIAllocator ObArenaAllocator(const lib::ObLabel &label = ObModIds::OB_MODULE_PAGE_ALLOCATOR, const int64_t page_size = OB_MALLOC_NORMAL_BLOCK_SIZE, int64_t ctx_id = 0) - : arena_(page_size, ModulePageAllocator(label, ctx_id)), tracker_(nullptr) {} - ObArenaAllocator(ObIAllocator &allocator, const int64_t page_size = OB_MALLOC_NORMAL_BLOCK_SIZE) - : arena_(page_size, ModulePageAllocator(allocator)), tracker_(nullptr) {}; + : arena_(page_size, ModulePageAllocator(label, ctx_id), true), tracker_(nullptr) {} + ObArenaAllocator(ObIAllocator &allocator, + const int64_t page_size = OB_MALLOC_NORMAL_BLOCK_SIZE, + const bool enable_sanity = false) + : arena_(page_size, ModulePageAllocator(allocator), enable_sanity), tracker_(nullptr) {}; ObArenaAllocator(const lib::ObMemAttr &attr, const int64_t page_size = OB_MALLOC_NORMAL_BLOCK_SIZE) - : arena_(page_size, ModulePageAllocator(attr)), tracker_(nullptr) {} + : arena_(page_size, ModulePageAllocator(attr), true), tracker_(nullptr) {} virtual ~ObArenaAllocator() { update_tracker(-arena_.total()); @@ -1140,7 +1294,7 @@ class ObAlignedArenaAllocator: public ObIAllocator const lib::ObLabel &label = ObModIds::OB_MODULE_PAGE_ALLOCATOR, const int64_t page_size = OB_MALLOC_NORMAL_BLOCK_SIZE, int64_t ctx_id = 0) - :arena_(page_size, ModulePageAllocator(label, ctx_id)), + :arena_(page_size, ModulePageAllocator(label, ctx_id), true), alignment_(alignment) {} virtual ~ObAlignedArenaAllocator() = default; diff --git a/src/oblib/lib/rc/context.h b/src/oblib/lib/rc/context.h index 0d3ccb083..039126e20 100644 --- a/src/oblib/lib/rc/context.h +++ b/src/oblib/lib/rc/context.h @@ -434,7 +434,8 @@ DISABLE_WARNING_GCC_POP ret = init_default_alloc(thread_safe, ablock_size); if (OB_SUCC(ret)) { // init arena allocator - p_arena_alloc_ = new (&arena_alloc_) common::ObArenaAllocator(*p_alloc_, param_.page_size_); + p_arena_alloc_ = new (&arena_alloc_) common::ObArenaAllocator( + *p_alloc_, param_.page_size_, true/*enable_sanity*/); arena_alloc_.set_attr(attr_); p_safe_arena_alloc_ = new (&safe_arena_alloc_) common::ObSafeArenaAllocator(arena_alloc_); default_allocator_ = (param_.properties_ & RETURN_MALLOC_DEFAULT) ? diff --git a/src/oblib/oblib_header_inventory.bzl b/src/oblib/oblib_header_inventory.bzl index 614d94131..ef7d3e3a8 100644 --- a/src/oblib/oblib_header_inventory.bzl +++ b/src/oblib/oblib_header_inventory.bzl @@ -749,6 +749,7 @@ OBLIB_HEADER_TARGETS = { "oblib_foundation": { "hdrs": [ "lib/geometry/ob_geo_wkb_define.h", + "lib/net/ob_sql_tls_info.h", "lib/ob_date_unit_type.h", "lib/ob_errno.h", "lib/ob_name_def.h", @@ -1644,6 +1645,7 @@ OBLIB_HEADER_TARGET_FOR_HEADER = { "lib/metrics/ob_counter.h": ":oblib_foundation_base", "lib/net/ob_addr.h": ":oblib_io_runtime", "lib/net/ob_net_util.h": ":oblib_io_runtime", + "lib/net/ob_sql_tls_info.h": ":oblib_foundation", "lib/ob_abort.h": ":oblib_foundation_base", "lib/ob_check_macros.h": ":oblib_foundation_base", "lib/ob_date_unit_type.h": ":oblib_foundation", diff --git a/src/oblib/oblib_source_inventory.bzl b/src/oblib/oblib_source_inventory.bzl index 67c7a701c..b7429fc0e 100644 --- a/src/oblib/oblib_source_inventory.bzl +++ b/src/oblib/oblib_source_inventory.bzl @@ -247,7 +247,9 @@ OBLIB_UNITY_GROUPS = { "src/oblib/lib/allocator/ob_delay_free_allocator.cpp", "src/oblib/lib/allocator/ob_fifo_allocator.cpp", "src/oblib/lib/allocator/ob_hazard_ref.cpp", + "src/oblib/lib/allocator/ob_jemalloc_sanity.cpp", "src/oblib/lib/allocator/ob_malloc.cpp", + "src/oblib/lib/allocator/ob_sanity_libc_wrap.cpp", "src/oblib/lib/utility/ob_mod_define.cpp", "src/oblib/lib/allocator/ob_page_manager.cpp", ], diff --git a/src/observer/BUILD.bazel b/src/observer/BUILD.bazel index 9bfca9d9c..4e3c0c36f 100644 --- a/src/observer/BUILD.bazel +++ b/src/observer/BUILD.bazel @@ -519,6 +519,12 @@ _SEEKDB_GRPC_ABSL_ARCHIVE_NAMES = [ _SEEKDB_ORDERED_LINK_FILES = [ "//rust:sql_nio_archive", +] + select({ + "//bazel:sanity_enabled": [ + "@seekdb_3rd_headers//:lib/libjemalloc_pic.a", + ], + "//conditions:default": [], +}) + [ "@seekdb_3rd_headers//:lib/libxml2.a", "@seekdb_3rd_headers//:lib/liblzma.a", "@seekdb_3rd_headers//:lib/vsag_lib/libcpuinfo.a", @@ -641,6 +647,29 @@ _SEEKDB_EXECUTABLE_LINKOPTS = select({ ], }) +_SEEKDB_SANITY_LINKOPTS = select({ + "//bazel:sanity_enabled": [ + "-Wl,--wrap=memcpy", + "-Wl,--wrap=memmove", + "-Wl,--wrap=memset", + "-Wl,--wrap=bzero", + "-Wl,--wrap=memcmp", + "-Wl,--wrap=strlen", + "-Wl,--wrap=strnlen", + "-Wl,--wrap=strcpy", + "-Wl,--wrap=strncpy", + "-Wl,--wrap=strcmp", + "-Wl,--wrap=strncmp", + "-Wl,--wrap=strcasecmp", + "-Wl,--wrap=strncasecmp", + "-Wl,--wrap=vsprintf", + "-Wl,--wrap=vsnprintf", + "-Wl,--wrap=sprintf", + "-Wl,--wrap=snprintf", + ], + "//conditions:default": [], +}) + _SEEKDB_SHARED_LINKOPTS = select({ "@platforms//os:macos": [ "-Wl,-framework,Security", @@ -674,7 +703,7 @@ seekdb_final_link( ordered_link_files = _SEEKDB_ORDERED_LINK_FILES, extra_inputs = _SEEKDB_EXTRA_LINK_INPUTS, system_libs = _SEEKDB_SYSTEM_LIBS, - linkopts = _SEEKDB_EXECUTABLE_LINKOPTS, + linkopts = _SEEKDB_EXECUTABLE_LINKOPTS + _SEEKDB_SANITY_LINKOPTS, out = "seekdb", tags = ["manual"], visibility = ["//visibility:public"], @@ -702,7 +731,7 @@ seekdb_final_link( ordered_link_files = _SEEKDB_ORDERED_LINK_FILES, extra_inputs = _SEEKDB_EXTRA_LINK_INPUTS, system_libs = _SEEKDB_SYSTEM_LIBS, - linkopts = _SEEKDB_SHARED_LINKOPTS, + linkopts = _SEEKDB_SHARED_LINKOPTS + _SEEKDB_SANITY_LINKOPTS, out = "liboceanbase.so", shared = True, tags = ["manual"], diff --git a/src/share/share_api_targets.bzl b/src/share/share_api_targets.bzl index b0e70e9d4..b5cca7bea 100644 --- a/src/share/share_api_targets.bzl +++ b/src/share/share_api_targets.bzl @@ -525,7 +525,10 @@ _SHARE_SEMANTIC_HEADER_TARGETS = { ], ), "runtime_context": struct( - hdrs = ["ob_server_struct.h"], + hdrs = [ + "ob_cpu_share_calculator.h", + "ob_server_struct.h", + ], deps = [ ":cluster_topology", ":config", diff --git a/src/share/share_header_inventory.bzl b/src/share/share_header_inventory.bzl index 40f6e5e46..6b030570c 100644 --- a/src/share/share_header_inventory.bzl +++ b/src/share/share_header_inventory.bzl @@ -94,6 +94,7 @@ SHARE_PUBLIC_HEADER_ROOTS = [ "ob_check_stop_provider.h", "ob_column_checksum_error_operator.h", "ob_common_id.h", + "ob_cpu_share_calculator.h", "ob_compatibility_control.h", "ob_core_table_proxy.h", "ob_ddl_checksum.h", @@ -375,7 +376,6 @@ SHARE_PRIVATE_HEADERS = [ "geo/ob_geo_vertex_collect_visitor.h", "geo/ob_point_location_analyzer.h", "geo/ob_wkb_to_json_visitor.h", - "ob_cpu_share_calculator.h", "ob_errno.def", "ob_i_mem_limit_getter.h", "ob_inner_kv_table_operator.h", diff --git a/tools/jemalloc_sanity_experiment/README.md b/tools/jemalloc_sanity_experiment/README.md new file mode 100644 index 000000000..6b96ea2d2 --- /dev/null +++ b/tools/jemalloc_sanity_experiment/README.md @@ -0,0 +1,68 @@ +# jemalloc + Sanity experiment + +This experiment routes jemalloc allocations through a dedicated arena whose +extent hook obtains memory from the address interval observed by Sanity. The +adapter keeps jemalloc metadata and redzones poisoned, and unpoisons only the +requested user bytes. + +The deleted OBMalloc implementation's maximum candidate was +`[0x0c0000000000, 0x600000000000)`, an 84 TiB application interval with a +10.5 TiB shadow interval; occupied mappings made it retreat in 128 GiB steps. +The first jemalloc proof of concept used only 64 GiB. This version restores the +former capacity policy: it tries the old upper bounds and retreats in the same +increments. Reservations use +`MAP_FIXED_NOREPLACE` (or a checked non-fixed hint on old kernels), so an +existing mapping is never overwritten. + +Run the focused checks after `./bazel.py deps init`: + +```sh +tools/jemalloc_sanity_experiment/run.sh valid +tools/jemalloc_sanity_experiment/run.sh overflow +tools/jemalloc_sanity_experiment/run.sh uaf +tools/jemalloc_sanity_experiment/run.sh memcpy_overflow +tools/jemalloc_sanity_experiment/run.sh snprintf_overflow +tools/jemalloc_sanity_experiment/run.sh sprintf_overflow +tools/jemalloc_sanity_experiment/run.sh arena_valid +tools/jemalloc_sanity_experiment/run.sh arena_reuse_valid +tools/jemalloc_sanity_experiment/run.sh arena_overflow +tools/jemalloc_sanity_experiment/run.sh arena_aligned_overflow +tools/jemalloc_sanity_experiment/run.sh arena_down_overflow +tools/jemalloc_sanity_experiment/run.sh arena_down_reuse_uaf +tools/jemalloc_sanity_experiment/run.sh arena_reuse_uaf +tools/jemalloc_sanity_experiment/run.sh arena_free_uaf +tools/jemalloc_sanity_experiment/run.sh arena_reset_remain_uaf +tools/jemalloc_sanity_experiment/run.sh arena_tracer_uaf +tools/jemalloc_sanity_experiment/run.sh arena_partial_free_uaf +tools/jemalloc_sanity_experiment/run.sh arena_partial_retrace_valid +tools/jemalloc_sanity_experiment/run.sh arena_aligned_bf_overflow +tools/jemalloc_sanity_experiment/run.sh arena_realloc_overflow +``` + +The `valid`, `arena_valid`, `arena_reuse_valid`, and +`arena_partial_retrace_valid` commands must exit successfully. The remaining +commands must stop in `memory_sanity_abort`. + +This is intentionally an experiment. jemalloc background threads and tcache +are disabled in Sanity mode. + +## Mapping from the former OBMalloc integration + +The old implementation had several independent adaptation layers. They map to +the current experiment as follows: + +| Former OBMalloc adaptation | jemalloc experiment | +| --- | --- | +| Reserve the application interval and its 1:8 shadow; allocate chunks with `sanity_mmap` | Reserve the same style of interval and give a dedicated jemalloc arena an extent hook backed by it | +| Poison `AObject` headers and allocation tails; unpoison only requested bytes; poison user bytes on free | Keep a private `AllocationHeader`, alignment padding, and redzone poisoned; unpoison only `requested_`; poison the full jemalloc allocation before release | +| Disable compiler range checks while allocator metadata is being manipulated | Build the allocator unity group without the Sanity pass and guard calls entering jemalloc | +| Add redzones inside `PageArena`, poison retained pages on reuse, and opt `MemoryContext` into that mode | `PageArena` has the same opt-in mode, plus coverage for `alloc_down`, best-fit aligned allocation, partial free, and tracer rollback; `MemoryContext` opts in explicitly | +| Unpoison/poison cache macroblocks around their OBMalloc lifecycle | The jemalloc backend's cache-macroblock path already calls `jemalloc_malloc/free`, so it inherits the common adapter | +| Sanity-aware libc operations supplied by the Sanity runtime | Final-link `--wrap` checks avoid that runtime's first-use `dlsym` recursion | + +The former direct `AChunk` mmap path is not copied into the normal jemalloc +allocation path because jemalloc replaces it there. Co-routine/thread-stack +chunks still use direct mappings and were intentionally excluded from shadow +allocation in the old code as well. The old SQL-operator datum checks were +proactive diagnostic checkpoints, not allocator correctness machinery, so +they are not restored by this experiment. diff --git a/tools/jemalloc_sanity_experiment/adapter_test.cpp b/tools/jemalloc_sanity_experiment/adapter_test.cpp new file mode 100644 index 000000000..6d437a6c5 --- /dev/null +++ b/tools/jemalloc_sanity_experiment/adapter_test.cpp @@ -0,0 +1,53 @@ +#include "lib/allocator/ob_jemalloc.h" + +#include +#include +#include + +extern int64_t sanity_min_addr; +extern int64_t sanity_max_addr; + +using oceanbase::common::jemalloc_sanity_free; +using oceanbase::common::jemalloc_sanity_malloc; + +__attribute__((noinline)) int run_case(const char *mode, char *ptr) { + if (0 == std::strcmp(mode, "overflow")) { + ptr[13] = 'x'; + } else if (0 == std::strcmp(mode, "uaf")) { + jemalloc_sanity_free(ptr); + ptr[0] = 'x'; + return 0; + } else if (0 == std::strcmp(mode, "memcpy_overflow")) { + char source[14] = {}; + std::memcpy(ptr, source, sizeof(source)); + } else if (0 == std::strcmp(mode, "snprintf_overflow")) { + std::snprintf(ptr, 32, "%s", "formatted output"); + } else if (0 == std::strcmp(mode, "sprintf_overflow")) { + std::sprintf(ptr, "%s", "formatted output"); + } else { + std::memset(ptr, 0x5a, 13); + if (0x5a != ptr[12]) { + return 3; + } + } + jemalloc_sanity_free(ptr); + return 0; +} + +int main(int argc, char **argv) { + const char *mode = argc > 1 ? argv[1] : "valid"; + char *ptr = static_cast(jemalloc_sanity_malloc(13)); + if (nullptr == ptr) { + std::fprintf(stderr, "allocation failed\n"); + return 2; + } + + std::printf("sanity range: [0x%llx, 0x%llx), %.2f TiB\n", + static_cast(sanity_min_addr), + static_cast(sanity_max_addr), + static_cast(sanity_max_addr - sanity_min_addr) / + static_cast(1ULL << 40)); + std::fflush(stdout); + + return run_case(mode, ptr); +} diff --git a/tools/jemalloc_sanity_experiment/page_arena_test.cpp b/tools/jemalloc_sanity_experiment/page_arena_test.cpp new file mode 100644 index 000000000..08f998d65 --- /dev/null +++ b/tools/jemalloc_sanity_experiment/page_arena_test.cpp @@ -0,0 +1,127 @@ +#include "lib/allocator/page_arena.h" + +#include +#include + +using oceanbase::common::PageArena; +using oceanbase::common::jemalloc_sanity_free; +using oceanbase::common::jemalloc_sanity_malloc; + +struct DirectPageAllocator +{ + void *alloc(const int64_t size) + { + return jemalloc_sanity_malloc(static_cast(size)); + } + void free(void *ptr) { jemalloc_sanity_free(ptr); } + void freed(const int64_t size) { static_cast(size); } +}; + +__attribute__((noinline)) int run_case(const char *mode) +{ + DirectPageAllocator page_allocator; + PageArena arena(4096, page_allocator, true); + char *first = arena.alloc(13); + if (nullptr == first) { + return 2; + } + + if (0 == std::strcmp(mode, "arena_overflow")) { + first[13] = 'x'; + } else if (0 == std::strcmp(mode, "arena_aligned_overflow")) { + char *aligned = arena.alloc_aligned(13, 64); + if (nullptr == aligned) { + return 5; + } + aligned[13] = 'x'; + } else if (0 == std::strcmp(mode, "arena_down_overflow")) { + char *down = arena.alloc_down(13); + if (nullptr == down) { + return 6; + } + down[13] = 'x'; + } else if (0 == std::strcmp(mode, "arena_down_reuse_uaf")) { + char *down = arena.alloc_down(13); + if (nullptr == down) { + return 7; + } + down[12] = 'x'; + arena.reuse(); + down[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_reuse_uaf")) { + arena.reuse(); + first[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_free_uaf")) { + arena.free(); + first[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_reset_remain_uaf")) { + arena.free_remain_one_page(); + first[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_tracer_uaf")) { + if (!arena.set_tracer()) { + return 8; + } + char *after_tracer = arena.alloc(13); + if (nullptr == after_tracer || !arena.revert_tracer()) { + return 9; + } + after_tracer[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_partial_free_uaf")) { + arena.partial_slow_free(0, 0, arena.total()); + first[0] = 'x'; + } else if (0 == std::strcmp(mode, "arena_partial_retrace_valid")) { + if (!arena.set_tracer()) { + return 10; + } + arena.partial_slow_free(0, 0, arena.total()); + if (!arena.set_tracer()) { + return 11; + } + char *after_partial_free = arena.alloc(13); + if (nullptr == after_partial_free) { + return 12; + } + std::memset(after_partial_free, 0x3d, 13); + if (0x3d != after_partial_free[12]) { + return 13; + } + } else if (0 == std::strcmp(mode, "arena_aligned_bf_overflow")) { + char *aligned = arena.alloc_aligned_bf(13, 64); + if (nullptr == aligned) { + return 14; + } + aligned[13] = 'x'; + } else if (0 == std::strcmp(mode, "arena_realloc_overflow")) { + char *grown = arena.realloc(first, 13, 29); + if (nullptr == grown) { + return 15; + } + grown[29] = 'x'; + } else if (0 == std::strcmp(mode, "arena_reuse_valid")) { + arena.reuse(); + char *reused = arena.alloc(13); + if (nullptr == reused) { + return 16; + } + std::memset(reused, 0x7c, 13); + if (0x7c != reused[12]) { + return 17; + } + } else { + char *second = arena.alloc(13); + if (nullptr == second) { + return 3; + } + std::memset(first, 0x5a, 13); + std::memset(second, 0x6b, 13); + if (0x5a != first[12] || 0x6b != second[12]) { + return 4; + } + } + return 0; +} + +int main(int argc, char **argv) +{ + return run_case(argc > 1 ? argv[1] : "arena_valid"); +} diff --git a/tools/jemalloc_sanity_experiment/run.sh b/tools/jemalloc_sanity_experiment/run.sh new file mode 100755 index 000000000..1151e6150 --- /dev/null +++ b/tools/jemalloc_sanity_experiment/run.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TOPDIR="$(cd "$(dirname "$0")/../.." && pwd)" +OUTDIR="${TOPDIR}/build_bazel/jemalloc_sanity_experiment" +DEPS="${TOPDIR}/deps/3rd/usr/local/oceanbase" +CLANG="${DEPS}/devtools/bin/clang++" +COMMON_FLAGS=( + -std=c++17 -g -O1 -Wno-inconsistent-missing-override + -DENABLE_SANITY -DOB_HAVE_BUNDLED_JEMALLOC=1 + -I"${TOPDIR}/src/oblib" + -I"${DEPS}/deps/devel/include" + -I"${DEPS}/devtools/include" +) +NO_BUILTIN_FLAGS=( + -fno-builtin-memcpy -fno-builtin-memmove -fno-builtin-memset + -fno-builtin-bzero -fno-builtin-memcmp + -fno-builtin-strlen -fno-builtin-strnlen + -fno-builtin-strcpy -fno-builtin-strncpy + -fno-builtin-strcmp -fno-builtin-strncmp + -fno-builtin-strcasecmp -fno-builtin-strncasecmp + -fno-builtin-vsprintf -fno-builtin-vsnprintf + -fno-builtin-sprintf -fno-builtin-snprintf +) +WRAP_FLAGS=( + -Wl,--wrap=memcpy -Wl,--wrap=memmove -Wl,--wrap=memset + -Wl,--wrap=bzero -Wl,--wrap=memcmp -Wl,--wrap=strlen + -Wl,--wrap=strnlen -Wl,--wrap=strcpy -Wl,--wrap=strncpy + -Wl,--wrap=strcmp -Wl,--wrap=strncmp + -Wl,--wrap=strcasecmp -Wl,--wrap=strncasecmp + -Wl,--wrap=vsprintf -Wl,--wrap=vsnprintf + -Wl,--wrap=sprintf -Wl,--wrap=snprintf +) + +mkdir -p "${OUTDIR}" +"${CLANG}" "${COMMON_FLAGS[@]}" -c \ + "${TOPDIR}/src/oblib/lib/allocator/ob_jemalloc_sanity.cpp" \ + -o "${OUTDIR}/adapter.o" +"${CLANG}" "${COMMON_FLAGS[@]}" -c \ + "${TOPDIR}/src/oblib/lib/allocator/ob_sanity_libc_wrap.cpp" \ + -o "${OUTDIR}/libc_wrap.o" +"${CLANG}" "${COMMON_FLAGS[@]}" \ + -fpass-plugin="${DEPS}/devtools/lib64/libsanitypass.so" \ + "${NO_BUILTIN_FLAGS[@]}" \ + -c "${TOPDIR}/tools/jemalloc_sanity_experiment/adapter_test.cpp" \ + -o "${OUTDIR}/adapter_test.o" +"${CLANG}" "${OUTDIR}/adapter_test.o" "${OUTDIR}/adapter.o" \ + "${OUTDIR}/libc_wrap.o" "${DEPS}/deps/devel/lib/libjemalloc_pic.a" \ + "${WRAP_FLAGS[@]}" -no-pie -pthread -ldl -o "${OUTDIR}/adapter_test" + +MODE="${1:-valid}" +if [[ "${MODE}" == arena_* ]]; then + "${CLANG}" "${COMMON_FLAGS[@]}" \ + -fpass-plugin="${DEPS}/devtools/lib64/libsanitypass.so" \ + "${NO_BUILTIN_FLAGS[@]}" \ + -c "${TOPDIR}/tools/jemalloc_sanity_experiment/page_arena_test.cpp" \ + -o "${OUTDIR}/page_arena_test.o" + "${CLANG}" "${OUTDIR}/page_arena_test.o" "${OUTDIR}/adapter.o" \ + "${OUTDIR}/libc_wrap.o" "${DEPS}/deps/devel/lib/libjemalloc_pic.a" \ + "${WRAP_FLAGS[@]}" -Wl,--gc-sections -Wl,--unresolved-symbols=ignore-all \ + -no-pie -pthread -ldl \ + -o "${OUTDIR}/page_arena_test" + "${OUTDIR}/page_arena_test" "${MODE}" +else + "${OUTDIR}/adapter_test" "${MODE}" +fi