From b318de4f7659362c041c2b1dcc5a572c28f5fcee Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:50:22 -0700 Subject: [PATCH 01/27] radlink: enable C11 atomics so BLAKE3 skips the lock'd CPU-feature load get_cpu_features was the top main-thread hot spot (~5.6s for one Fortnite link), 97% of it inside a single ATOMIC_LOAD(g_cpu_features). On MSVC, blake3_dispatch.c defines ATOMIC_LOAD as _InterlockedOr(&x,0) -- a lock'd RMW (full barrier) run on every BLAKE3 compress dispatch. The value is written once and read-only after, so the barrier is pointless. Enable BLAKE3's plain-load path (C11 _Atomic, a plain mov on x86) via build flags only, leaving the vendored third_party/blake3 source untouched: /std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1 Scoped to the radlink target. MSVC C11 atomics need both /std:c11 and /experimental:c11atomics. get_cpu_features: 5591ms -> 4ms (main thread). Co-Authored-By: Claude Opus 4.8 (1M context) --- build.bat | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build.bat b/build.bat index 5c9c71d46..54f70b945 100644 --- a/build.bat +++ b/build.bat @@ -141,7 +141,14 @@ popd pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +:: NOTE: -DBLAKE3_ATOMICS=1 makes BLAKE3 use C11 _Atomic (plain atomic load) for +:: get_cpu_features instead of MSVC's _InterlockedOr `lock or` barrier on every +:: compress dispatch (was a ~5.5s main-thread hot spot). MSVC C11 atomics require +:: /std:c11 /experimental:c11atomics. Kept external so the vendored blake3 source +:: stays pristine. +set radlink_msvc_flags= +if "%msvc%"=="1" set radlink_msvc_flags=/std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1 +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_msvc_flags% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 From 9aa79c350e79a47abcf4e7e6ca6d6e7723b9a25f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:52:33 -0700 Subject: [PATCH 02/27] radlink: skip symbol-name string-table scan on interp-only paths coff_read_symbol_name scans a cstr in the memory-mapped string table -- the dominant, page-fault-bound cost of bulk symbol parsing. Many hot callers parse a full symbol but only read scalar fields (value/section/storage_class/aux) to interpret the symbol value; the name is never used. Add name-skipping parse variants and route the interp-only paths through them: coff_parse_symbol{16,32}_no_name (coff_parse.c) -- and the full variants now call these + add the name, so the scalar logic lives in one place lnk_parsed_symbol_from_coff_symbol_idx_no_name (lnk_obj.c) lnk_interp_from_symbol / lnk_can_replace_symbol / lnk_on_symbol_replace (lnk_symbol_table.c) and the lnk_search_lib_task loop (lnk.c) Where the name is still needed (lnk_search_lib) it uses the already-cached LNK_Symbol.name instead of re-parsing. lnk_can_replace_symbol previously parsed dst/src twice (full parse + a second parse for interp); collapsed to one no-name parse each. coff_parse_symbol32 on the main thread: 3922ms -> ~810ms (name-needed callers). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coff/coff_parse.c | 28 ++++++++++++++++++++++++---- src/coff/coff_parse.h | 2 ++ src/linker/lnk.c | 7 ++++--- src/linker/lnk_obj.c | 19 ++++++++++++++++++- src/linker/lnk_obj.h | 1 + src/linker/lnk_symbol_table.c | 19 ++++++++++++------- 6 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/coff/coff_parse.c b/src/coff/coff_parse.c index 1db0b5826..85835b977 100644 --- a/src/coff/coff_parse.c +++ b/src/coff/coff_parse.c @@ -166,11 +166,16 @@ coff_section_header_array_from_name(Arena *arena, String8 string_table, COFF_Sec } +// NOTE: name-skipping variants. coff_read_symbol_name does a cstr scan over the +// memory-mapped string table, which is the dominant cost when parsing symbols in +// bulk; callers that only need the scalar fields (value/section/storage_class/aux, +// e.g. symbol-value interpretation) should use these to avoid that scan. The full +// coff_parse_symbol{16,32} below are these plus the name read, so the scalar-field +// logic lives in exactly one place. internal COFF_ParsedSymbol -coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +coff_parse_symbol32_no_name(COFF_Symbol32 *sym32) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym32->name); result.value = sym32->value; result.section_number = sym32->section_number; result.type = sym32->type; @@ -181,10 +186,9 @@ coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) } internal COFF_ParsedSymbol -coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +coff_parse_symbol16_no_name(COFF_Symbol16 *sym16) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym16->name); result.value = sym16->value; if (sym16->section_number == COFF_Symbol_DebugSection16) { result.section_number = COFF_Symbol_DebugSection32; @@ -200,6 +204,22 @@ coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) return result; } +internal COFF_ParsedSymbol +coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +{ + COFF_ParsedSymbol result = coff_parse_symbol32_no_name(sym32); + result.name = coff_read_symbol_name(string_table, &sym32->name); + return result; +} + +internal COFF_ParsedSymbol +coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +{ + COFF_ParsedSymbol result = coff_parse_symbol16_no_name(sym16); + result.name = coff_read_symbol_name(string_table, &sym16->name); + return result; +} + internal COFF_ParsedSymbol coff_parse_symbol(COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx) { diff --git a/src/coff/coff_parse.h b/src/coff/coff_parse.h index edcb29520..028e292f5 100644 --- a/src/coff/coff_parse.h +++ b/src/coff/coff_parse.h @@ -261,6 +261,8 @@ internal String8 coff_name_from_section_header (String8 str internal COFF_ParsedSymbol coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32); internal COFF_ParsedSymbol coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16); +internal COFF_ParsedSymbol coff_parse_symbol32_no_name(COFF_Symbol32 *sym32); +internal COFF_ParsedSymbol coff_parse_symbol16_no_name(COFF_Symbol16 *sym16); internal COFF_ParsedSymbol coff_parse_symbol (COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx); internal COFF_Symbol32Array coff_symbol_array_from_data_16(Arena *arena, String8 data, U64 symbol_array_off, U64 symbol_count); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 250a33c45..a95348ca9 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1623,7 +1623,8 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) LNK_Symbol *symbol = c->v[i].symbol; LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // skip the name scan here; the resolved name is already cached on symbol->name + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); if (symbol_interp == COFF_SymbolValueInterp_Undefined) { U32 member_idx; @@ -1641,11 +1642,11 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) if (search_anti_deps) { LNK_ObjSymbolRef dep_symbol = {0}; if (lnk_resolve_weak_symbol(symtab, symbol_ref, &dep_symbol)) { - COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx(dep_symbol.obj, dep_symbol.symbol_idx); + COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dep_symbol.obj, dep_symbol.symbol_idx); COFF_SymbolValueInterpType dep_interp = coff_interp_from_parsed_symbol(dep_parsed); if (dep_interp == COFF_SymbolValueInterp_Weak) { U32 member_idx; - if (lnk_search_lib(lib, symbol_parsed.name, &member_idx)) { + if (lnk_search_lib(lib, symbol->name, &member_idx)) { lnk_queue_lib_member(arena, task->imports_hm, task->link->lib_member_infos_hm, member_ref_list, symbol, lib, lib_member_infos, member_idx); } } diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index dabd77f83..84bb5b039 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -659,7 +659,24 @@ lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) } else { result = coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); } - + + return result; +} + +// NOTE: same as above but skips the symbol-name string-table scan; use when only +// the scalar fields are needed (e.g. symbol-value interpretation). +internal COFF_ParsedSymbol +lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) +{ + String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); + + COFF_ParsedSymbol result = {0}; + if (obj->header.is_big_obj) { + result = coff_parse_symbol32_no_name((COFF_Symbol32 *)symbol_table.str + symbol_idx); + } else { + result = coff_parse_symbol16_no_name((COFF_Symbol16 *)symbol_table.str + symbol_idx); + } + return result; } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index d13a3d1e1..874441aca 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -146,6 +146,7 @@ internal LNK_Symbol * lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_n internal COFF_SectionHeader * lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx); +internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx); internal U64 lnk_obj_sect_idx_from_section_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx); diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 6cd8734ef..ec018035a 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -105,12 +105,14 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) { B32 can_replace = 0; - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_ParsedSymbol src_parsed = lnk_parsed_from_symbol(src); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); - COFF_SymbolValueInterpType src_interp = lnk_interp_from_symbol(src); + // only scalar fields + raw_symbol are needed here (names come from dst->name/src->name); + // skip the symbol-name string-table scan and avoid re-parsing for interp LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); LNK_ObjSymbolRef src_ref = lnk_ref_from_symbol(src); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_ParsedSymbol src_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(src_ref.obj, src_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); + COFF_SymbolValueInterpType src_interp = coff_interp_from_parsed_symbol(src_parsed); LNK_Obj *dst_obj = dst_ref.obj; LNK_Obj *src_obj = src_ref.obj; @@ -332,9 +334,10 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) internal void lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) { - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); + // only scalar fields are needed below, so skip the symbol-name string-table scan LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); if (dst_interp == COFF_SymbolValueInterp_Regular) { // remove replaced section from the output @@ -532,7 +535,9 @@ lnk_parsed_from_symbol(LNK_Symbol *symbol) internal COFF_SymbolValueInterpType lnk_interp_from_symbol(LNK_Symbol *symbol) { - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // interp only needs scalar fields; skip the symbol-name string-table scan + LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref.obj, ref.symbol_idx); return coff_interp_from_parsed_symbol(symbol_parsed); } From f97b1ebedc6bf2826ce562e6a8180815366d3c4f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:55:05 -0700 Subject: [PATCH 03/27] radlink: make symbol ref-list merge O(1) with a tail pointer lnk_on_symbol_replace merged ref lists by walking the destination's singly linked refs list to its tail on every merge. Across repeated COMDAT merges into one accumulating leader this is O(n^2) and was 96% of the function. Add a refs_tail pointer to LNK_Symbol so the append is O(1): src->refs_tail->next = dst->refs; src->refs_tail = dst->refs_tail; maintained at all ref-list write sites (lnk_make_symbol, the null_symbol and import-stub sites in lnk.c). Order and head identity are preserved exactly, so this is a pure perf change: the head node stays the primary ref, and interior order is irrelevant (every multi-ref consumer sorts). lnk_on_symbol_replace (main thread): 1306ms -> 161ms exclusive. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 4 +++- src/linker/lnk_symbol_table.c | 8 ++++---- src/linker/lnk_symbol_table.h | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index a95348ca9..0d4e4e7f0 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1785,6 +1785,7 @@ lnk_link_inputs(TP_Context *tp, if (null_symbol == 0) { null_symbol = push_array(inputer->arena, LNK_Symbol, 1); null_symbol->refs = push_array(inputer->arena, LNK_ObjSymbolRefNode, 1); + null_symbol->refs_tail = null_symbol->refs; null_symbol->refs->v.obj = &link->objs.first->data; } LNK_LibMemberRef *member_refs = push_array(scratch.arena, LNK_LibMemberRef, lib->member_count); @@ -1880,7 +1881,8 @@ lnk_link_inputs(TP_Context *tp, AssertAlways(member_ref->link_symbol->refs != import_stub->refs); // replace the import symbol with a stub, which is later replaced with the real import symbol once import obj is ready. - member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs_tail = import_stub->refs_tail; // push import member for import obj generation lnk_lib_member_ref_list_push_node(&link->imports, member_ref); diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index ec018035a..4822f9246 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -11,6 +11,7 @@ lnk_make_symbol(Arena *arena, String8 name, LNK_Obj *obj, U32 symbol_idx) LNK_Symbol *symbol = push_array(arena, LNK_Symbol, 1); symbol->name = name; symbol->refs = ref; + symbol->refs_tail = ref; return symbol; } @@ -353,10 +354,9 @@ lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) } } - // merge symbol refs - LNK_ObjSymbolRefNode *src_last_ref; - for (src_last_ref = src->refs; src_last_ref->next != 0; src_last_ref = src_last_ref->next); - src_last_ref->next = dst->refs; + // merge symbol refs (append dst's list onto src's tail; tail pointer keeps this O(1)) + src->refs_tail->next = dst->refs; + src->refs_tail = dst->refs_tail; // assert leader section is live #if BUILD_DEBUG diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index cdf383045..b36a2ba64 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -21,6 +21,7 @@ typedef struct LNK_Symbol { String8 name; LNK_ObjSymbolRefNode *refs; + LNK_ObjSymbolRefNode *refs_tail; // tail of `refs`, so symbol-ref merges are O(1) instead of O(n) tail walks } LNK_Symbol; // --- Symbol Containers ------------------------------------------------------- From d052062295ede246c8a3d4d55487b1f61a3511bf Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:56:10 -0700 Subject: [PATCH 04/27] thread_pool: wake workers with one batched semaphore release tp_for_parallel woke workers with a loop of single semaphore_drop calls -- one ReleaseSemaphore syscall per worker (twice in shared mode). The main thread spent ~3.3s in ReleaseSemaphore over a Fortnite link. Add semaphore_drop_n(sem, count) (a single ReleaseSemaphore(h, count, 0) on Windows; a loop on POSIX) and wake all drop_count workers in one call. Two details keep the batched release correct: - Wake the full drop_count (NOT drop_count-1). The main thread runs as worker 0, but tasks that nest a tp_broadcast_ barrier span all workers; under-waking by one leaves that barrier a participant short and deadlocks on small dispatches. - Give the exec/task semaphores 2x max-count headroom. A single batched release can land while up to worker_count-1 previously-woken workers have not yet re-taken their permit, so the count can transiently approach 2*worker_count; a tight max would make ReleaseSemaphore fail outright and deadlock at the next barrier. ReleaseSemaphore (main thread): 3274ms -> 940ms. --- src/base/base_threads.h | 1 + src/linker/thread_pool/thread_pool.c | 31 ++++++++++++++++------------ src/linux/base/linux_base.c | 13 ++++++++++++ src/win32/base/win32_base.c | 9 ++++++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/base/base_threads.h b/src/base/base_threads.h index b34286ccb..9e156d593 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -129,6 +129,7 @@ internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); internal void semaphore_drop(Semaphore semaphore); +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 71003bb98..180edb169 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -68,12 +68,18 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) Semaphore exec_semaphore = {0}; if (worker_count > 1) { main_semaphore = semaphore_alloc(0, 1, str8_zero()); + // Max counts carry 2x headroom: tp_for_parallel wakes workers with a single + // batched ReleaseSemaphore(drop_count); a batch can land while up to + // worker_count-1 previously-woken workers have not yet re-taken their permit, + // so the count can transiently reach ~2*worker_count. A tight max (== worker + // count) would make that batched release exceed the max and fail outright, + // waking no one and deadlocking at the next barrier. if (is_shared) { AssertAlways(worker_count <= max_worker_count); - task_semaphore = semaphore_alloc(0, max_worker_count, name); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * max_worker_count, name); + exec_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } else { - task_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } } @@ -210,18 +216,17 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF U64 drop_count = Min(task_count, pool->worker_count); - // if we are in shared mode ping local semaphore + // Wake exactly drop_count workers in a single batched ReleaseSemaphore. The + // count MUST be drop_count (not drop_count-1): tasks that nest tp_broadcast_ + // (e.g. lnk_walk_relocs_and_mark_ref_sections_task) barrier across all + // worker_count participants, and under-waking by one leaves the barrier short + // -> deadlock. Overflow from the batch is prevented by the 2x semaphore max + // headroom set in tp_alloc. if (pool->exec_semaphore.u64[0] != 0) { - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx +=1) { - semaphore_drop(pool->exec_semaphore); - } - } - - // ping shared semaphore - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx += 1) { - semaphore_drop(pool->task_semaphore); + semaphore_drop_n(pool->exec_semaphore, (U32)drop_count); } - + semaphore_drop_n(pool->task_semaphore, (U32)drop_count); + // run tasks on main worker tp_run_tasks(pool, &pool->worker_arr[0]); diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index d96751388..bcbe6cc04 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -707,6 +707,19 @@ semaphore_drop(Semaphore semaphore) } } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + //- rjf: barriers internal Barrier diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index 799452038..35db175f1 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -692,6 +692,15 @@ semaphore_drop(Semaphore semaphore) ReleaseSemaphore(handle, 1, 0); } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + ReleaseSemaphore(handle, count, 0); + } +} + //- rjf: barriers internal Barrier From ee7dc9e8e8e3192a26f0c17233b542313c9a3218 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:57:06 -0700 Subject: [PATCH 05/27] base: use memchr in str8_cstring_capped instead of a byte loop The capped cstr length scan was a byte-by-byte loop. Switch to memchr, which is SIMD-accelerated in the CRT. Speeds up every capped-cstr scan in the codebase; notably cv_name_from_symbol (CodeView symbol-name scan during GSI build). cv_name_from_symbol (main thread): 1098ms -> 201ms. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/base/base_strings.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/base/base_strings.c b/src/base/base_strings.c index 8825d4798..78e3e59ac 100644 --- a/src/base/base_strings.c +++ b/src/base/base_strings.c @@ -213,7 +213,9 @@ str8_cstring_capped(void *cstr, void *cap) { char *ptr = (char *)cstr; char *opl = (char *)cap; - for (;ptr < opl && *ptr != 0; ptr += 1); + // memchr is typically SIMD-accelerated; much faster than a byte-by-byte scan + char *nul = (char *)memchr(ptr, 0, (U64)(opl - ptr)); + ptr = nul ? nul : opl; U64 size = (U64)(ptr - (char *)cstr); String8 result = str8((U8*)cstr, size); return result; From 05af760b44c1803fccdf9b739cea960c648cfba2 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 14:58:11 -0700 Subject: [PATCH 06/27] radlink: fold CV type-index fixup into a single hash probe lnk_fixup_cv_type_indices did two open-addressing probes per type-index reference: lnk_leaf_hash_table_search (leaf_ref -> canonical bucket), then lnk_assigned_type_ht_search (canonical bucket -> assigned type index, via a second hash table keyed by leaf-ref content). Both are cache-miss-bound and this ran across every type-index reference in every obj. Store the assigned type index directly on the leaf hash table: add a ti_arr parallel to bucket_arr. lnk_assign_type_indices_task writes ti = min+i into the leaf's bucket slot (each unique leaf owns a distinct slot, so worker writes never collide), and the new lnk_leaf_hash_table_search_ti recovers it in one probe. Removes the entire assigned_type_hts table and its build pass; deletes the now-dead lnk_leaf_hash_table_search and lnk_assigned_type_ht_search. Correctness: deduplicated leaves share the same ghash (debug_h value), so the fixup query and the assign-time canonical bucket hash to the same slot. lnk_fixup_cv_type_indices (main thread): 1445ms -> 390ms inclusive. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 112 +++++++++++------------------------- src/linker/lnk_debug_info.h | 5 +- 2 files changed, 36 insertions(+), 81 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 83f574969..c5187af10 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1374,12 +1374,13 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } -internal LNK_LeafRef * -lnk_leaf_hash_table_search(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +// Probes the leaf hash table for leaf_ref's canonical (deduped) bucket and returns +// the assigned type index stored on that slot (ti_arr), or 0 if absent. Folds what +// used to be two lookups (canonical-bucket lookup, then a separate bucket->type-index +// hash lookup) into a single probe. +internal CV_TypeIndex +lnk_leaf_hash_table_search_ti(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - LNK_LeafRef *match = 0; - - CV_DebugT *debug_t = &input->debug_t_arr[leaf_ref.obj_idx]; CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; U64 hash = debug_h->v[leaf_ref.leaf_idx]; U64 best_bucket_idx = hash % ht->cap; @@ -1389,14 +1390,13 @@ lnk_leaf_hash_table_search(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_ if (bucket == 0) { break; } if (lnk_match_leaf_ref(input, *bucket, leaf_ref)) { - match = bucket; - break; + return ht->ti_arr[bucket_idx]; } bucket_idx = (bucket_idx + 1) == ht->cap ? 0 : (bucket_idx + 1); } while (bucket_idx != best_bucket_idx); - return match; + return 0; } internal @@ -1748,57 +1748,38 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) { LNK_MergeTypes *task = raw_task; - CV_TypeIndexSource ti_source = task->ti_source; - LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; - CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; - U64 assigned_type_cap = task->assigned_type_caps[ti_source]; - CV_TypeIndex *assigned_type_ht = task->assigned_type_hts[ti_source]; + CV_TypeIndexSource ti_source = task->ti_source; + LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; + CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; + LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[ti_source]; + CV_DebugH *debug_h_arr = task->input->debug_h_arr; + // Store each unique leaf's assigned type index directly on its bucket slot in + // the leaf hash table, so later fixups recover the type index with a single + // probe (lnk_leaf_hash_table_search_ti) instead of a second hash lookup. + // Each unique leaf owns a distinct slot, so writes never collide across workers. for EachInRange(i, task->ranges[task_id]) { LNK_LeafRef *leaf_ref = unique_leaf_refs.v[i]; CV_TypeIndex type_index = min_type_index + i; - U64 hash = u64_hash_from_str8(str8_struct(leaf_ref)); - U64 best_idx = hash % assigned_type_cap; + U64 hash = debug_h_arr[leaf_ref->obj_idx].v[leaf_ref->leaf_idx]; + U64 best_idx = hash % leaf_ht->cap; U64 idx = best_idx; - B32 is_inserted = 0; + B32 is_assigned = 0; do { - CV_TypeIndex curr_type_index = assigned_type_ht[idx]; - if (curr_type_index == 0) { - CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(&assigned_type_ht[idx], type_index, curr_type_index); - if (cmp_type_index == curr_type_index) { - is_inserted = 1; - break; - } + if (leaf_ht->bucket_arr[idx] == leaf_ref) { + leaf_ht->ti_arr[idx] = type_index; + is_assigned = 1; + break; } // advance - idx = (idx + 1) == assigned_type_cap ? 0 : (idx + 1); + idx = (idx + 1) == leaf_ht->cap ? 0 : (idx + 1); } while (idx != best_idx); - Assert(is_inserted); + Assert(is_assigned); } } -internal CV_TypeIndex -lnk_assigned_type_ht_search(U64 cap, CV_TypeIndex *ht, CV_TypeIndex min_type_index, LNK_LeafRefArray unique_leaf_refs, LNK_LeafRef *v, U64 hash) -{ - U64 best_idx = hash % cap; - U64 idx = best_idx; - do { - CV_TypeIndex type_index = ht[idx]; - if (type_index < min_type_index) { break; } - - U64 leaf_idx = type_index - min_type_index; - LNK_LeafRef *compar = unique_leaf_refs.v[leaf_idx]; - if (MemoryMatchStruct(compar,v)) { return type_index; } - - idx = (idx + 1) == cap ? 0 : (idx + 1); - } while(idx != best_idx); - - InvalidPath; - return 0; -} - internal void lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) { @@ -1809,21 +1790,11 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ // skip basic types if (ti < ctx->input->min_type_indices[n->source]) { continue; } - CV_TypeIndex final_ti = 0; - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, ctx->input, leaf_ref); - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - final_ti = lnk_assigned_type_ht_search(ctx->assigned_type_caps [n->source], - ctx->assigned_type_hts [n->source], - ctx->min_type_indices [n->source], - ctx->unique_leaf_refs_arr[n->source], - final_leaf, - final_hash); - } + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); + LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; + CV_TypeIndex final_ti = lnk_leaf_hash_table_search_ti(leaf_ht, ctx->input, leaf_ref); #if BUILD_DEBUG - else { + if (final_ti == 0) { lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); } #endif @@ -1992,23 +1963,9 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) for EachIndex(leaf_idx, debug_t->count) { CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); - LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; - LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, input, leaf_ref); - - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - CV_TypeIndex final_ti = lnk_assigned_type_ht_search(task->assigned_type_caps [source], - task->assigned_type_hts [source], - task->min_type_indices [source], - task->unique_leaf_refs_arr[source], - final_leaf, - final_hash); - - obj_ti_map[leaf_idx] = final_ti; - } else { - obj_ti_map[leaf_idx] = 0; - } + LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; + LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[source]; + obj_ti_map[leaf_idx] = lnk_leaf_hash_table_search_ti(leaf_ht, input, leaf_ref); } task->result.obj_ti_maps[obj_idx] = obj_ti_map; @@ -2096,6 +2053,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_ht_arr[ti_source].cap = total_count; task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 task.leaf_ht_arr[ti_source].bucket_arr = push_array(scratch.arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); + task.leaf_ht_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); @@ -2250,8 +2208,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ProfBegin("Assign type indices"); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; - task.assigned_type_caps[ti_source] = (task.unique_leaf_refs_arr[ti_source].count * 13) / 10; - task.assigned_type_hts [ti_source] = push_array(scratch.arena, CV_TypeIndex, task.assigned_type_caps[ti_source]); task.min_type_indices [ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_assign_type_indices_task, &task, "Assign Type Indices"); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index dc094e0f9..8543aeef4 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -117,6 +117,7 @@ typedef struct { U64 cap; LNK_LeafRef **bucket_arr; + CV_TypeIndex *ti_arr; // assigned type index per bucket slot (parallel to bucket_arr); 0 = unassigned } LNK_LeafHashTable; typedef struct LNK_LeafRange @@ -174,8 +175,6 @@ typedef struct U64 pass_idx; // assign type indices - U64 assigned_type_caps [CV_TypeIndexSource_COUNT]; - CV_TypeIndex *assigned_type_hts [CV_TypeIndexSource_COUNT]; CV_TypeIndex min_type_indices [CV_TypeIndexSource_COUNT]; LNK_LeafRefArray unique_leaf_refs_arr[CV_TypeIndexSource_COUNT]; @@ -251,7 +250,7 @@ internal B32 lnk_match_leaf_ref (LNK_CodeViewInput internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); -internal LNK_LeafRef * lnk_leaf_hash_table_search (LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal CV_TypeIndex lnk_leaf_hash_table_search_ti (LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); // returns assigned ti for leaf_ref's canonical bucket, 0 if absent internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); From 0e9b3febcdb19d587a37a3d74ade98687b244893 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 16:05:00 -0700 Subject: [PATCH 07/27] radlink: release copy-on-write input views in parallel before exit Input obj/lib files are mapped copy-on-write (PAGE_WRITECOPY/FILE_MAP_COPY) so the linker can patch them in place. Pages touched during linking become private-dirty; at process exit the kernel reclaims them in single-threaded address-space rundown -- ~3s of lingering process time after the last thread exits for a large (Fortnite-scale) link. After all outputs are written and inputs are no longer read (post image-write join), unmap the whole-file CoW views in parallel on the thread pool. The same reclaim work then runs multi-threaded, off the serial post-exit path: measured ~34s of aggregate UnmapViewOfFile CPU collapsing to ~0.55s wall, and the post-exit process tail dropping from ~3s to ~0.5s. Only the is_thin whole-file views are swept (lib-member substrings and linkgen arena data are skipped), and only in the copy-on-write (read-only) mapping mode -- read-write-shared mapping would flush dirty pages back to the input files on unmap. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 0d4e4e7f0..e83fa1091 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -5265,6 +5265,7 @@ lnk_write_thread(void *raw_ctx) ProfEnd(); } + internal void lnk_log_timers(void) { @@ -5353,6 +5354,52 @@ lnk_debug_filter_objs(Arena *arena, LNK_Obj **objs, U64 objs_count, U64 *count_o return debug_info_objs; } +// Parallel release of memory-mapped input file views. +// Inputs are mapped copy-on-write (PAGE_WRITECOPY/FILE_MAP_COPY); pages touched +// during linking become private-dirty and are reclaimed by the kernel in +// single-threaded process rundown at exit (~3s for a large link). Unmapping them +// in parallel before exit moves that reclaim off the serial post-exit path. +typedef struct LNK_UnmapViewTask +{ + String8 *views; +} LNK_UnmapViewTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_unmap_view_task) +{ + LNK_UnmapViewTask *task = raw_task; + String8 view = task->views[task_id]; +#if OS_WINDOWS + UnmapViewOfFile(view.str); +#elif OS_LINUX + munmap(view.str, view.size); +#endif +} + +internal void +lnk_release_input_views(TP_Context *tp, LNK_Inputer *inputer) +{ + Temp scratch = scratch_begin(0, 0); + + // collect distinct whole-file mapped views (is_thin); skip lib-member + // substrings and linkgen arena data + U64 cap = inputer->objs.count + inputer->libs.count; + String8 *views = push_array_no_zero(scratch.arena, String8, cap); + U64 count = 0; + for EachNode(n, LNK_Input, inputer->objs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + for EachNode(n, LNK_Input, inputer->libs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + + if (count > 0) { + U64 begin_us = now_time_us(); + LNK_UnmapViewTask task = { .views = views }; + tp_for_parallel(tp, 0, count, lnk_unmap_view_task, &task); + U64 end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "Released %llu input views in %.2f ms", count, (F64)(end_us - begin_us) / 1000.0); + } + + scratch_end(scratch); +} + internal void lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) { @@ -5548,6 +5595,16 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // outputs are written and inputs are no longer read; release the copy-on-write + // input views in parallel so their dirty pages are reclaimed here (multi-threaded) + // instead of in single-threaded process rundown at exit. Only safe for the CoW + // (read-only) mapping mode; read-write-shared would flush dirty pages back to the + // input files on unmap. + if ((config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + lnk_release_input_views(tp, inputer); + } + // // Timers // From 8fcd84b9cbb27795b1b7894ed653b1df51422eaa Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Tue, 16 Jun 2026 20:04:48 -0700 Subject: [PATCH 08/27] radlink: trim COFF symbol/section parse overhead Two micro-optimizations on hot parse helpers (profiled as the largest aggregate-CPU functions in a Fortnite link): - lnk_obj_section_from_sect_idx: split out a _no_name variant that skips the section-name string-table lookup (coff_name_from_section_header). The full variant now reuses it + adds the name. lnk_raw_directives_from_obj iterated every section of every obj building the full section struct just to test a flag, computing the name on ~all sections though only .drectve needs it -- now uses the no-name variant and resolves the name only inside the LnkInfo branch. - lnk_parsed_symbol_from_coff_symbol_idx (+_no_name): return the coff_parse_* result directly instead of zero-initializing a local and assigning to it, removing a redundant ~48-byte COFF_ParsedSymbol copy + zero-init per call (RVO). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_obj.c | 32 +++++++++++++++++++------------- src/linker/lnk_obj.h | 1 + 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 84bb5b039..7f5e67fcd 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -570,8 +570,11 @@ lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx) return sect_idx+1; } +// NOTE: skips section.name (coff_name_from_section_header does a string-table +// lookup); use when only the header/flags/ranges are needed (e.g. checking +// section flags). Callers that need the name should use lnk_obj_section_from_sect_idx. internal LNK_ObjSection -lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx) { Assert(sect_idx < obj->header.section_count_no_null); LNK_ObjSection section = {0}; @@ -580,13 +583,20 @@ lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) section.section_number = sect_idx+1; section.header = &lnk_coff_section_table_from_obj(obj)[sect_idx]; section.flags = &obj->section_flags[sect_idx]; - section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); section.vrange = rng_1u64(section.header->voff, section.header->voff + section.header->vsize); section.frange = rng_1u64(section.header->foff, section.header->foff + section.header->fsize); section.reloc_count = section.header->reloc_count; return section; } +internal LNK_ObjSection +lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +{ + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); + return section; +} + internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number) { @@ -653,14 +663,11 @@ lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) String8 string_table = str8_substr(obj->data, obj->header.string_table_range); String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - COFF_ParsedSymbol result = {0}; if (obj->header.is_big_obj) { - result = coff_parse_symbol32(string_table, (COFF_Symbol32 *)symbol_table.str + symbol_idx); + return coff_parse_symbol32(string_table, (COFF_Symbol32 *)symbol_table.str + symbol_idx); } else { - result = coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); + return coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); } - - return result; } // NOTE: same as above but skips the symbol-name string-table scan; use when only @@ -670,14 +677,11 @@ lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) { String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - COFF_ParsedSymbol result = {0}; if (obj->header.is_big_obj) { - result = coff_parse_symbol32_no_name((COFF_Symbol32 *)symbol_table.str + symbol_idx); + return coff_parse_symbol32_no_name((COFF_Symbol32 *)symbol_table.str + symbol_idx); } else { - result = coff_parse_symbol16_no_name((COFF_Symbol16 *)symbol_table.str + symbol_idx); + return coff_parse_symbol16_no_name((COFF_Symbol16 *)symbol_table.str + symbol_idx); } - - return result; } internal @@ -773,8 +777,10 @@ lnk_raw_directives_from_obj(Arena *arena, LNK_Obj *obj) { String8List drectve_data = {0}; for (U64 sect_idx = 0; sect_idx < obj->header.section_count_no_null; sect_idx += 1) { - LNK_ObjSection section = lnk_obj_section_from_sect_idx(obj, sect_idx); + // only LnkInfo sections (rare) need the name; skip the string-table lookup for the rest + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); if (*section.flags & COFF_SectionFlag_LnkInfo) { + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); if (str8_match(section.name, str8_lit(".drectve"), 0)) { if (*section.flags & COFF_SectionFlag_CntUninitializedData) { lnk_error_obj(LNK_Error_IllData, obj, ".drectve section header has flag COFF_SectionFlag_CntUninitializedData"); diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 874441aca..0e1c5cf17 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -150,6 +150,7 @@ internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK internal U64 lnk_obj_sect_idx_from_section_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx); +internal LNK_ObjSection lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); From 5d400d2dde19dbdef6f0195e52827fcab57ca648 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 10:29:32 -0700 Subject: [PATCH 09/27] radlink: cache leaf hash per bucket to avoid deref in type-dedup probe lnk_leaf_hash_table_search_ti (~#2 radlink hotspot, ~48% of its self-time) spent its time in lnk_match_leaf_ref, which is just a_hash==b_hash but fetches the bucket's hash via lnk_hash_from_leaf_ref -> input->debug_h_arr[obj].v[leaf], a scattered cache miss per probe step. Add LNK_LeafHashTable.hash_arr (parallel to bucket_arr), populated at bucket claim/update (both lnk_populate_leaf_ht and lnk_leaf_dedup_task) with the leaf's debug_h hash. search_ti now matches via hash_arr[idx] == hash -- no deref. Exact equivalent (match is pure hash compare); same value across same-hash updates. Gated 65/65 linker torture (ghash_basic/match_debug_t, determ_test, p2r_determinism). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 7 ++++++- src/linker/lnk_debug_info.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index c5187af10..c12bc3130 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1389,7 +1389,9 @@ lnk_leaf_hash_table_search_ti(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, L LNK_LeafRef *bucket = ht->bucket_arr[bucket_idx]; if (bucket == 0) { break; } - if (lnk_match_leaf_ref(input, *bucket, leaf_ref)) { + // match by cached hash (== lnk_match_leaf_ref, which is just a_hash==b_hash) without dereferencing + // *bucket into the scattered debug_h_arr + if (ht->hash_arr[bucket_idx] == hash) { return ht->ti_arr[bucket_idx]; } @@ -1473,6 +1475,7 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) // try to update the bucket LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); if (cmp == curr) { + leaf_ht->hash_arr[idx] = debug_h->v[leaf_idx]; // cache occupant's hash for deref-free search_ti bucket = 0; goto exit; } @@ -1528,6 +1531,7 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) // try to update the bucket LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); if (cmp == curr) { + leaf_ht->hash_arr[idx] = debug_h->v[leaf_idx]; // cache occupant's hash for deref-free search_ti bucket = 0; goto exit; } @@ -2054,6 +2058,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 task.leaf_ht_arr[ti_source].bucket_arr = push_array(scratch.arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); task.leaf_ht_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.leaf_ht_arr[ti_source].cap); + task.leaf_ht_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 8543aeef4..2e3bd8c0b 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -118,6 +118,8 @@ typedef struct U64 cap; LNK_LeafRef **bucket_arr; CV_TypeIndex *ti_arr; // assigned type index per bucket slot (parallel to bucket_arr); 0 = unassigned + U64 *hash_arr; // leaf hash of the occupying bucket (parallel to bucket_arr); lets search_ti + // match by hash without dereferencing *bucket into the scattered debug_h_arr } LNK_LeafHashTable; typedef struct LNK_LeafRange From be3e6635de3685722810b983f2db348a233b377e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 11:14:28 -0700 Subject: [PATCH 10/27] radlink: release the ~1GB image buffer early so reclaim overlaps the run The image buffer was push'd on the shared link arena and only reclaimed in the single-threaded process rundown at exit -- a multi-second kernel page-reclaim tail (observed: one thread 100% in-kernel, zero user frames). Allocate it as a standalone reserve_memory/commit_memory region and release_memory() it the instant the background image-write thread joins (image is on disk, no later reader). VirtualFree(MEM_RELEASE) returns fast; the kernel zeroes the ~1GB on its background thread, overlapping the parallel input-view release + exit instead of blocking rundown. Discard early so the kernel cleans up while the app still runs -- don't defer to exit. Gated 65/65 linker torture (determ_test + p2r_determinism: image correct). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index e83fa1091..b1b926d1c 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -4746,7 +4746,11 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); @@ -5595,6 +5599,10 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); + // outputs are written and inputs are no longer read; release the copy-on-write // input views in parallel so their dirty pages are reclaimed here (multi-threaded) // instead of in single-threaded process rundown at exit. Only safe for the CoW From 68c5ef3a619c73b8039b95dde70296caf66cdb33 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 15:28:04 -0700 Subject: [PATCH 11/27] radlink: memoize parsed COFF symbols per obj (kills #1 link hotspot) Parse each COFF symbol once in lnk_obj_initer into LNK_Obj.parsed_symbols; lnk_parsed_symbol_from_coff_symbol_idx[_no_name] becomes an array index instead of re-decoding the mmapped symbol table on every access (it was the #1 hotspot, lnk_parsed_symbol_from_coff_symbol_idx). All symbol-value patch sites (weak-replace, COMDAT-leader, regular/common fixups) write obj->parsed_symbols[idx], decoupling symbol values from the input mapping. Extracted from the entangled WIP commit 8dc5fa68 (parsed-symbol memo + .rgd staging); this is the memo half only -- no .rgd. Gated separately. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 95 ++++++++--------------------------- src/linker/lnk_obj.c | 28 ++++------- src/linker/lnk_obj.h | 2 + src/linker/lnk_symbol_table.c | 14 ++---- 4 files changed, 36 insertions(+), 103 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index b1b926d1c..0eafb8439 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2953,15 +2953,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_comdat_leaders_task) task->u.patch_symtabs.was_symbol_patched[obj_idx][symbol_idx] = 1; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } } @@ -3025,15 +3018,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_leaders_task) COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol); U64 section_number = task->u.patch_symtabs.common_block_sect->sect_idx + 1; - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = parsed_symbol.raw_symbol; - symbol32->value = contrib->u.offset; - symbol32->section_number = safe_cast_u32(section_number); - } else { - COFF_Symbol16 *symbol16 = parsed_symbol.raw_symbol; - symbol16->value = contrib->u.offset; - symbol16->section_number = safe_cast_u16(section_number); - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = contrib->u.offset; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = safe_cast_u32(section_number); task->u.patch_symtabs.was_symbol_patched[symbol_ref.obj->input_idx][symbol_ref.symbol_idx] = 1; } @@ -3058,17 +3044,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_symbols_task) COFF_ParsedSymbol defn_parsed = lnk_parsed_from_symbol(defn); Assert(lnk_interp_from_symbol(defn) == COFF_SymbolValueInterp_Regular); if (defn) { - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = defn_parsed.section_number; - symbol32->value = safe_cast_u32(defn_parsed.value); - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(defn_parsed.section_number); - symbol16->value = safe_cast_u32(defn_parsed.value); - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = defn_parsed.section_number; + obj->parsed_symbols[symbol_idx].value = defn_parsed.value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } } } @@ -3106,15 +3084,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_regular_symbols_task) value = sc->u.off + symbol.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(section_number); - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } ProfEnd(); @@ -3151,17 +3122,9 @@ lnk_patch_obj_symtab(LNK_SymbolTable *symtab, LNK_Obj *obj, B8 *was_symbol_patch value = fixup_src.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = fixup_dst.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = fixup_dst.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; was_symbol_patched[symbol_idx] = 1; } @@ -3842,17 +3805,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) if (sect && (~sect->flags & COFF_SectionFlag_LnkRemove)) { if (~sect->flags & COFF_SectionFlag_MemDiscardable) { LNK_SectionContrib *first_sc = lnk_get_first_section_contrib(sect); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = safe_cast_u32(first_sc->u.sect_idx + 1); - symbol32->value = first_sc->u.off; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(first_sc->u.sect_idx + 1); - symbol16->value = first_sc->u.off; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = safe_cast_u32(first_sc->u.sect_idx + 1); + obj->parsed_symbols[symbol_idx].value = first_sc->u.off; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } else { lnk_error_obj(LNK_Error_SectRefsDiscardedMemory, obj, "symbol %S (No. 0x%llx) references section with discard flag", symbol.name, symbol_idx); } @@ -3870,17 +3825,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) LNK_Section *fallback_sect = task->image_sects.v[task->image_sects.count-1]; U32 fallback_section_number = safe_cast_u32(fallback_sect->sect_idx + 1); U32 fallback_section_offset = safe_cast_u32(fallback_voff - fallback_sect->voff); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = fallback_section_number; - symbol32->value = fallback_section_offset; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(fallback_section_number); - symbol16->value = fallback_section_offset; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = fallback_section_number; + obj->parsed_symbols[symbol_idx].value = fallback_section_offset; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; lnk_error_obj(LNK_Warning_UndefinedSectionSymbol, obj, "undefined section symbol %S (No. 0x%llx) refers to an image section that doesn't exist; patching to %#llx", symbol.name, symbol_idx, fallback_voff); } @@ -5422,7 +5369,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_SymbolTable *symtab = lnk_symbol_table_init(arena); // - // Link Image + // Link Image (group digests, if any, are synthesized + consumed inside lnk_link_image) // LNK_LinkResult link = lnk_link_image(tp, arena, config, inputer, symtab); @@ -5907,8 +5854,8 @@ entry_point(CmdLine *cmdline) } switch (config->boot_mode) { - case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; - case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; + case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; + case LNK_BootMode_TypeServer: lnk_run_type_server (tp, tp_arena, config); break; } lnk_log_end(); diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 7f5e67fcd..790140c26 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -125,13 +125,15 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } // - // error check symbol table + // error check symbol table (+ memoize parsed symbols) // + COFF_ParsedSymbol *parsed_symbols = push_array(arena, COFF_ParsedSymbol, header.symbol_count); { COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(input->data, header.section_table_range).str; COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = coff_parse_symbol(header, raw_coff_string_table, raw_coff_symbol_table, symbol_idx); + parsed_symbols[symbol_idx] = symbol; COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); if (interp == COFF_SymbolValueInterp_Regular) { if (symbol.section_number == 0 || symbol.section_number > header.section_count_no_null) { @@ -335,6 +337,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) obj->path = push_str8_copy(arena, input->path); obj->header = header; obj->section_flags = section_flags; + obj->parsed_symbols = parsed_symbols; obj->comdats = comdats; obj->exclude_from_debug_info = input->exclude_from_debug_info; obj->hotpatch = hotpatch; @@ -657,31 +660,20 @@ lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number) return §ion_table[sect_idx]; } +// NOTE: returns the memoized parse built in lnk_obj_initer. The struct is mutable: symbol-value +// patching (lnk_patch_*_task) writes value/section_number/storage_class here, NOT into the mmapped +// obj->data symbol table -- so obj->data symbol values are never written after load. raw_symbol still +// points into obj->data (read-only) for aux-record reads (coff_parse_weak_tag / coff_parse_secdef). internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) { - String8 string_table = str8_substr(obj->data, obj->header.string_table_range); - String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - - if (obj->header.is_big_obj) { - return coff_parse_symbol32(string_table, (COFF_Symbol32 *)symbol_table.str + symbol_idx); - } else { - return coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); - } + return obj->parsed_symbols[symbol_idx]; } -// NOTE: same as above but skips the symbol-name string-table scan; use when only -// the scalar fields are needed (e.g. symbol-value interpretation). internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) { - String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - - if (obj->header.is_big_obj) { - return coff_parse_symbol32_no_name((COFF_Symbol32 *)symbol_table.str + symbol_idx); - } else { - return coff_parse_symbol16_no_name((COFF_Symbol16 *)symbol_table.str + symbol_idx); - } + return obj->parsed_symbols[symbol_idx]; } internal diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 0e1c5cf17..055674ce0 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -12,6 +12,8 @@ typedef struct LNK_Obj COFF_FileHeaderInfo header; COFF_SectionFlags *section_flags; + COFF_ParsedSymbol *parsed_symbols; // memoized parse per symbol_idx (aux slots zeroed). Mutable: symbol-value + // patching writes here, NOT into the mmapped obj->data symbol table. // flags B8 hotpatch; diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 4822f9246..d1d117d08 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -778,17 +778,9 @@ THREAD_POOL_TASK_FUNC(lnk_replace_weak_with_default_symbol_task) COFF_SymbolValueInterpType resolve_interp = coff_interp_from_parsed_symbol(resolve_parsed); if (resolve_interp == COFF_SymbolValueInterp_Weak) { COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(resolve_parsed, symbol_ref.obj->header.is_big_obj); - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol_parsed.raw_symbol; - symbol32->section_number = COFF_Symbol_UndefinedSection; - symbol32->value = 0; - symbol32->storage_class = COFF_SymStorageClass_External; - } else { - COFF_Symbol16 *symbol16 = symbol_parsed.raw_symbol; - symbol16->section_number = COFF_Symbol_UndefinedSection; - symbol16->value = 0; - symbol16->storage_class = COFF_SymStorageClass_External; - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = COFF_Symbol_UndefinedSection; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = 0; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].storage_class = COFF_SymStorageClass_External; } else { symbol->refs->v = resolve; } From 4e077fd9233e7b873f4375f3d554c5b9e6579731 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 16:03:06 -0700 Subject: [PATCH 12/27] radlink: size the assigned-ti table by unique types, not total (reclaim ~3GB peak) The CV type-index fixup (05af760b) had stored the assigned type index in arrays parallel to the dedup hash table (leaf_ht), whose cap is the TOTAL pre-dedup leaf count summed over all objs. On large links that ti_arr (+ the hash_arr added for deref-free probing) added ~3GB to peak working set vs the prior unique-sized assigned_type_ht. Split the two concerns: - LNK_LeafHashTable: just {cap, bucket_arr} for dedup (total-sized, as before / unavoidable). - LNK_AssignedTiHash {cap, ti_arr, hash_arr}: hash -> assigned ti, sized to the UNIQUE (post-dedup) leaf count. Built in lnk_assign_type_indices_task by hashing each unique leaf into its own slot (atomic claim; unique leaves have distinct hashes since dedup is by hash). search_ti probes it in one deref-free pass (occupant hash stored on the slot), exactly as before -- just on a table sized by unique instead of total. Keeps the single-probe fixup speed of 05af760b; removes its peak-memory regression. ti==0 marks empty (assigned ti is always >= CV_MinComplexTypeIndex). Builds clean, 95 linker torture PASS, 0 fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 79 +++++++++++++++++-------------------- src/linker/lnk_debug_info.h | 18 ++++++--- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index c12bc3130..87aad99d9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1374,29 +1374,21 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } -// Probes the leaf hash table for leaf_ref's canonical (deduped) bucket and returns -// the assigned type index stored on that slot (ti_arr), or 0 if absent. Folds what -// used to be two lookups (canonical-bucket lookup, then a separate bucket->type-index -// hash lookup) into a single probe. +// Returns the type index assigned to leaf_ref's deduped class, or 0 if absent. Dedup is by hash +// (lnk_match_leaf_ref is a_hash==b_hash), so the assigned-ti table is keyed purely by hash -- a single +// probe, deref-free (the occupant hash is on the slot), and sized to the UNIQUE leaf count. internal CV_TypeIndex -lnk_leaf_hash_table_search_ti(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +lnk_leaf_hash_table_search_ti(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; - U64 hash = debug_h->v[leaf_ref.leaf_idx]; - U64 best_bucket_idx = hash % ht->cap; - U64 bucket_idx = best_bucket_idx; + CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; + U64 hash = debug_h->v[leaf_ref.leaf_idx]; + U64 best_idx = hash % ht->cap; + U64 idx = best_idx; do { - LNK_LeafRef *bucket = ht->bucket_arr[bucket_idx]; - if (bucket == 0) { break; } - - // match by cached hash (== lnk_match_leaf_ref, which is just a_hash==b_hash) without dereferencing - // *bucket into the scattered debug_h_arr - if (ht->hash_arr[bucket_idx] == hash) { - return ht->ti_arr[bucket_idx]; - } - - bucket_idx = (bucket_idx + 1) == ht->cap ? 0 : (bucket_idx + 1); - } while (bucket_idx != best_bucket_idx); + if (ht->ti_arr[idx] == 0) { break; } // empty slot -> not present + if (ht->hash_arr[idx] == hash) { return ht->ti_arr[idx]; } + idx = (idx + 1) == ht->cap ? 0 : (idx + 1); + } while (idx != best_idx); return 0; } @@ -1475,7 +1467,6 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) // try to update the bucket LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); if (cmp == curr) { - leaf_ht->hash_arr[idx] = debug_h->v[leaf_idx]; // cache occupant's hash for deref-free search_ti bucket = 0; goto exit; } @@ -1531,7 +1522,6 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) // try to update the bucket LNK_LeafRef *cmp = ins_atomic_ptr_eval_cond_assign(&leaf_ht->bucket_arr[idx], bucket, curr); if (cmp == curr) { - leaf_ht->hash_arr[idx] = debug_h->v[leaf_idx]; // cache occupant's hash for deref-free search_ti bucket = 0; goto exit; } @@ -1755,30 +1745,31 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) CV_TypeIndexSource ti_source = task->ti_source; LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; - LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[ti_source]; + LNK_AssignedTiHash *at = &task->assigned_ti_arr[ti_source]; CV_DebugH *debug_h_arr = task->input->debug_h_arr; - // Store each unique leaf's assigned type index directly on its bucket slot in - // the leaf hash table, so later fixups recover the type index with a single - // probe (lnk_leaf_hash_table_search_ti) instead of a second hash lookup. - // Each unique leaf owns a distinct slot, so writes never collide across workers. + // Insert each unique leaf's assigned type index into the (unique-sized) hash->ti table, keyed by leaf + // hash. Unique leaves have distinct hashes (dedup is by hash), so each claims its own empty slot. + // search_ti (in the later fixup phase, after this barrier) recovers ti in one deref-free probe. for EachInRange(i, task->ranges[task_id]) { LNK_LeafRef *leaf_ref = unique_leaf_refs.v[i]; CV_TypeIndex type_index = min_type_index + i; U64 hash = debug_h_arr[leaf_ref->obj_idx].v[leaf_ref->leaf_idx]; - U64 best_idx = hash % leaf_ht->cap; + U64 best_idx = hash % at->cap; U64 idx = best_idx; B32 is_assigned = 0; do { - if (leaf_ht->bucket_arr[idx] == leaf_ref) { - leaf_ht->ti_arr[idx] = type_index; - is_assigned = 1; - break; + if (at->ti_arr[idx] == 0) { + CV_TypeIndex cmp = ins_atomic_u32_eval_cond_assign(&at->ti_arr[idx], type_index, 0); + if (cmp == 0) { + at->hash_arr[idx] = hash; // only this worker owns the slot now; read back in the fixup phase + is_assigned = 1; + break; + } } - // advance - idx = (idx + 1) == leaf_ht->cap ? 0 : (idx + 1); + idx = (idx + 1) == at->cap ? 0 : (idx + 1); } while (idx != best_idx); Assert(is_assigned); } @@ -1794,9 +1785,9 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ // skip basic types if (ti < ctx->input->min_type_indices[n->source]) { continue; } - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; - CV_TypeIndex final_ti = lnk_leaf_hash_table_search_ti(leaf_ht, ctx->input, leaf_ref); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); + LNK_AssignedTiHash *assigned = &ctx->assigned_ti_arr[n->source]; + CV_TypeIndex final_ti = lnk_leaf_hash_table_search_ti(assigned, ctx->input, leaf_ref); #if BUILD_DEBUG if (final_ti == 0) { lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); @@ -1967,9 +1958,9 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) for EachIndex(leaf_idx, debug_t->count) { CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); - LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; - LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[source]; - obj_ti_map[leaf_idx] = lnk_leaf_hash_table_search_ti(leaf_ht, input, leaf_ref); + LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; + LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; + obj_ti_map[leaf_idx] = lnk_leaf_hash_table_search_ti(assigned, input, leaf_ref); } task->result.obj_ti_maps[obj_idx] = obj_ti_map; @@ -2057,8 +2048,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.leaf_ht_arr[ti_source].cap = total_count; task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 task.leaf_ht_arr[ti_source].bucket_arr = push_array(scratch.arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); - task.leaf_ht_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.leaf_ht_arr[ti_source].cap); - task.leaf_ht_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); @@ -2115,6 +2104,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.unique_leaf_refs_arr[ti_source].count = sum_array_u64(tp->worker_count, task.counts[ti_source]); task.unique_leaf_refs_arr[ti_source].v = push_array_no_zero(scratch.arena, LNK_LeafRef *, task.unique_leaf_refs_arr[ti_source].count); + + // assigned-ti table sized to the unique (deduped) count -- not the total leaf count, which would + // add the bucket-parallel ti/hash arrays' worth of peak working set (~3GB on large links) + task.assigned_ti_arr[ti_source].cap = 1 + ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); // * 1.3 + task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); + task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 2e3bd8c0b..aa49eee42 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -115,13 +115,20 @@ typedef struct { U64 count; LNK_LeafRef **v; } LNK_LeafRefArray; typedef struct { - U64 cap; + U64 cap; // ~1.3x total (pre-dedup) leaf count LNK_LeafRef **bucket_arr; - CV_TypeIndex *ti_arr; // assigned type index per bucket slot (parallel to bucket_arr); 0 = unassigned - U64 *hash_arr; // leaf hash of the occupying bucket (parallel to bucket_arr); lets search_ti - // match by hash without dereferencing *bucket into the scattered debug_h_arr } LNK_LeafHashTable; +// Maps leaf hash -> assigned type index, sized to the UNIQUE (post-dedup) leaf count rather than the +// total leaf count -- folds the canonical-bucket + bucket->ti lookups into one probe (deref-free: the +// hash is stored on the slot) without the total-sized per-bucket arrays that would add ~3GB to peak. +typedef struct +{ + U64 cap; // ~1.3x unique leaf count + CV_TypeIndex *ti_arr; // assigned type index per slot; 0 == empty (ti is always >= CV_MinComplexTypeIndex) + U64 *hash_arr; // occupant leaf hash (parallel to ti_arr); disambiguates open-addressing collisions +} LNK_AssignedTiHash; + typedef struct LNK_LeafRange { struct LNK_LeafRange *next; @@ -153,6 +160,7 @@ typedef struct LNK_CodeViewInput *input; CV_DebugS *debug_s_arr; LNK_LeafHashTable leaf_ht_arr[CV_TypeIndexSource_COUNT]; + LNK_AssignedTiHash assigned_ti_arr[CV_TypeIndexSource_COUNT]; Arena **fixed_arenas; CV_TypeIndexSource ti_source; U32Array indices; @@ -252,7 +260,7 @@ internal B32 lnk_match_leaf_ref (LNK_CodeViewInput internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); -internal CV_TypeIndex lnk_leaf_hash_table_search_ti (LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); // returns assigned ti for leaf_ref's canonical bucket, 0 if absent +internal CV_TypeIndex lnk_leaf_hash_table_search_ti (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); // returns assigned ti for leaf_ref's hash class, 0 if absent internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); From c2c3018c0f9272f4ffc8112b139bcb9704c319c3 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Wed, 17 Jun 2026 19:00:59 -0700 Subject: [PATCH 13/27] radlink: accept GNU ar (.a) archives as lib input g_input_type_map mapped o/obj/lib/rlib/res/rrt but not .a, so clang/meson-built GNU ar archives (e.g. ThirdParty libdav1d.a) hit Error(002) 'unknown file format'. rlib (also GNU ar) already routes to LNK_Input_Lib and parses, so map .a the same. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 3d27012c3..9aa8caee0 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -119,6 +119,7 @@ global read_only struct { "obj", LNK_Input_Obj }, { "lib", LNK_Input_Lib }, { "rlib", LNK_Input_Lib }, // rust libs + { "a", LNK_Input_Lib }, // GNU ar archives (clang/meson, e.g. libdav1d.a) { "res", LNK_Input_Res }, { "rrt", LNK_Input_RRT }, }; From 4fac1f674e8d91244ac47a3bad2e1af6d4ca38ce Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 18 Jun 2026 10:53:56 -0700 Subject: [PATCH 14/27] radlink: slim the parsed-symbol memo (drop name, decode on demand) LNK_Obj.parsed_symbols memoized the full COFF_ParsedSymbol per symbol -- including the 16B String8 name -- sized by total symbol count, held to exit. Store a slim LNK_ParsedSymbolLite (every field except name, ~24B vs ~40B) and re-decode the name from the read-only symbol record in the named accessor only. The hot _no_name path (can_replace/GC/resolution) and all symbol-value patching never touch the name, so they stay fully memoized; only the named/push path pays a re-decode (cold relative to total). FN no-rrt full link: peak commit 50.5GB -> 49.1GB (-1.4GB), wall flat (~8-9s no-debug), valid 5.68GB PDB, 95/0 linker torture. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_obj.c | 28 ++++++++++++++++++++++------ src/linker/lnk_obj.h | 18 ++++++++++++++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 790140c26..8897ee5ce 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -127,13 +127,13 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) // // error check symbol table (+ memoize parsed symbols) // - COFF_ParsedSymbol *parsed_symbols = push_array(arena, COFF_ParsedSymbol, header.symbol_count); + LNK_ParsedSymbolLite *parsed_symbols = push_array(arena, LNK_ParsedSymbolLite, header.symbol_count); { COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(input->data, header.section_table_range).str; COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = coff_parse_symbol(header, raw_coff_string_table, raw_coff_symbol_table, symbol_idx); - parsed_symbols[symbol_idx] = symbol; + parsed_symbols[symbol_idx] = (LNK_ParsedSymbolLite){ symbol.value, symbol.section_number, symbol.type, symbol.storage_class, symbol.aux_symbol_count, symbol.raw_symbol }; COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); if (interp == COFF_SymbolValueInterp_Regular) { if (symbol.section_number == 0 || symbol.section_number > header.section_count_no_null) { @@ -665,15 +665,31 @@ lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number) // obj->data symbol table -- so obj->data symbol values are never written after load. raw_symbol still // points into obj->data (read-only) for aux-record reads (coff_parse_weak_tag / coff_parse_secdef). internal COFF_ParsedSymbol -lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) +lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) { - return obj->parsed_symbols[symbol_idx]; + LNK_ParsedSymbolLite *lite = &obj->parsed_symbols[symbol_idx]; + COFF_ParsedSymbol result = {0}; + result.value = lite->value; + result.section_number = lite->section_number; + result.type = lite->type; + result.storage_class = lite->storage_class; + result.aux_symbol_count = lite->aux_symbol_count; + result.raw_symbol = lite->raw_symbol; + return result; } internal COFF_ParsedSymbol -lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) +lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) { - return obj->parsed_symbols[symbol_idx]; + COFF_ParsedSymbol result = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, symbol_idx); + // name is excluded from the memo -- decode it from the (read-only) symbol record on demand. Patching + // never touches the name, so re-deriving from raw_symbol stays correct after value/section patches. + if (result.raw_symbol) { + String8 string_table = str8_substr(obj->data, obj->header.string_table_range); + if (obj->header.is_big_obj) { result.name = coff_parse_symbol32(string_table, (COFF_Symbol32 *)result.raw_symbol).name; } + else { result.name = coff_parse_symbol16(string_table, (COFF_Symbol16 *)result.raw_symbol).name; } + } + return result; } internal diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 055674ce0..0fa40b0bb 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -5,6 +5,20 @@ // --- Input ------------------------------------------------------------------- +// Slim memoized symbol parse: every COFF_ParsedSymbol field EXCEPT name (the 16B String8). Sized by +// total symbol count, so dropping name saves ~16B/sym of peak. The name is the cold path -- it is +// re-decoded from raw_symbol on demand in lnk_parsed_symbol_from_coff_symbol_idx (the named accessor); +// the hot _no_name accessor and all symbol-value patching never touch it. +typedef struct LNK_ParsedSymbolLite +{ + U64 value; + U32 section_number; + COFF_SymbolType type; + COFF_SymStorageClass storage_class; + U8 aux_symbol_count; + void *raw_symbol; +} LNK_ParsedSymbolLite; + typedef struct LNK_Obj { String8 path; @@ -12,8 +26,8 @@ typedef struct LNK_Obj COFF_FileHeaderInfo header; COFF_SectionFlags *section_flags; - COFF_ParsedSymbol *parsed_symbols; // memoized parse per symbol_idx (aux slots zeroed). Mutable: symbol-value - // patching writes here, NOT into the mmapped obj->data symbol table. + LNK_ParsedSymbolLite *parsed_symbols; // memoized parse per symbol_idx (aux slots zeroed), name excluded. + // Mutable: symbol-value patching writes here, NOT obj->data. // flags B8 hotpatch; From fe5204ecc41572188ee659675a0d9e58d12ef6fe Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Thu, 18 Jun 2026 12:22:29 -0700 Subject: [PATCH 15/27] radlink: pack LNK_ParsedSymbolLite to 16B (offset + U32 value) Store the COFF symbol record as a U32 byte-offset into obj->data instead of an 8B pointer, and value as U32 (COFF symbol value is U32). Struct goes 24B -> 16B (no padding): raw_symbol_off(4) value(4) section_number(4) type(2) storage_class(1) aux(1). The named/no_name accessors reconstruct the pointer as obj->data.str + off (obj->data == input->data, so the offset is stable). Sized by total symbol count -> 8B/sym off peak. FN no-rrt full link: peak commit 49.1GB -> 48.4GB (-0.67GB), wall flat, valid 5.68GB PDB, 95/0 linker torture. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_obj.c | 5 +++-- src/linker/lnk_obj.h | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 8897ee5ce..fcc541c5e 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -133,7 +133,8 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = coff_parse_symbol(header, raw_coff_string_table, raw_coff_symbol_table, symbol_idx); - parsed_symbols[symbol_idx] = (LNK_ParsedSymbolLite){ symbol.value, symbol.section_number, symbol.type, symbol.storage_class, symbol.aux_symbol_count, symbol.raw_symbol }; + U32 raw_off = symbol.raw_symbol ? safe_cast_u32((U8 *)symbol.raw_symbol - input->data.str) : 0; + parsed_symbols[symbol_idx] = (LNK_ParsedSymbolLite){ raw_off, safe_cast_u32(symbol.value), symbol.section_number, symbol.type, symbol.storage_class, symbol.aux_symbol_count }; COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); if (interp == COFF_SymbolValueInterp_Regular) { if (symbol.section_number == 0 || symbol.section_number > header.section_count_no_null) { @@ -674,7 +675,7 @@ lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) result.type = lite->type; result.storage_class = lite->storage_class; result.aux_symbol_count = lite->aux_symbol_count; - result.raw_symbol = lite->raw_symbol; + result.raw_symbol = lite->raw_symbol_off ? (obj->data.str + lite->raw_symbol_off) : 0; // offset -> ptr return result; } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 0fa40b0bb..95df7bde7 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -11,12 +11,13 @@ // the hot _no_name accessor and all symbol-value patching never touch it. typedef struct LNK_ParsedSymbolLite { - U64 value; + U32 raw_symbol_off; // byte offset of the COFF symbol record within obj->data (0 = none). + // stored as an offset, not a pointer, to keep this struct 16B. + U32 value; // COFF symbol value is U32 (section-relative offset / size / etc.) U32 section_number; - COFF_SymbolType type; - COFF_SymStorageClass storage_class; + COFF_SymbolType type; // U16 + COFF_SymStorageClass storage_class; // U8 U8 aux_symbol_count; - void *raw_symbol; } LNK_ParsedSymbolLite; typedef struct LNK_Obj From a78313a65cbd4d770154582eb123777ac6136a9f Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 09:31:29 -0700 Subject: [PATCH 16/27] radlink: garbage-collect unreferenced CodeView types before PDB emit After type merging, prune any merged TPI/IPI record not transitively reachable from a surviving symbol. Roots are the type indices referenced by the symbols that survive /OPT:REF (plus inlinee call-site types); the type graph is then closed over and everything unreached is dropped before the streams are written. Runs only under /OPT:REF -- it is the debug-info analogue of dead-section stripping, and is otherwise transparent (no visible type is removed). Implementation notes: - parallel transitive closure (bulk-synchronous rounds, atomic mark/expand) - fwdref<->definition pairing via a per-unique-name ring so a live forward reference keeps its definition (and vice-versa) - compaction is in place with the remap kept in scratch, so peak memory is unchanged Numbers (UnrealEditorFortnite-Engine.dll, /OPT:REF /OPT:ICF, hashing NONE): PDB 5315 -> 5081 MB (type-GC alone: -234 MB) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 290 ++++++++++++++++++++++++++++++++++++ src/linker/lnk_debug_info.h | 1 + 2 files changed, 291 insertions(+) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 87aad99d9..5e42f4877 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3091,6 +3091,296 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task) } } +//////////////////////////////// +// Type Garbage Collection +// +// After type merging, prune merged TPI/IPI leaves that are not reachable from any surviving +// symbol record (the GC roots), compact them, and remap all type indices. link.exe keeps a +// large unreferenced-type set; pruning it is a transparent PDB-size win (debug-info only -- the +// image is untouched). Runs on the final post-fixup type indices in place. + +typedef struct LNK_GCTypes +{ + LNK_CodeViewInput *cv; + U64 min [CV_TypeIndexSource_COUNT]; // first type index per source + U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source + U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) + U8 *expanded[CV_TypeIndexSource_COUNT]; // transitive-closure: already-visited bitmap + CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index + U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays + U32 *udt_next; // TPI fwdref<->definition unique_name ring + U32 *changed; // set when a closure round marks something new + Rng1U64 *sym_ranges; + B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // per-source scratch (set before dispatch) + CV_TypeIndexSource cur_source; + U8 **cur_leaf_v; + Rng1U64 *cur_ranges; +} LNK_GCTypes; + +typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; + +internal int +lnk_gc_name_pair_is_before(void *raw_a, void *raw_b) +{ + LNK_GCNamePair *a = raw_a, *b = raw_b; + return a->hash != b->hash ? (a->hash < b->hash) : (a->idx < b->idx); +} + +internal void +lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) +{ + U64 lo = g->min[s]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[s]) { g->mark[s][idx] = 1; } } +} + +// walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) +internal void +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TypeIndexInfoList ti_info_list) +{ + for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { + U8 *p = data.str + n->offset; + CV_TypeIndex ti = memory_read32(p); + if (g->do_rewrite) { + U64 lo = g->min[n->source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n->source]) { memory_write32(p, g->remap[n->source][idx]); } } + } else { + lnk_gc_mark_ti(g, n->source, ti); + } + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->sym_ranges[task_id]) { + LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + Temp temp = temp_begin(scratch.arena); + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + CV_TypeIndexInfoList l = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); + lnk_gc_visit_offsets(g, symbol.data, l); + temp_end(temp); + } + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) +{ + LNK_GCTypes *g = raw_task; + U64 obj_idx = task_id; + Temp scratch = scratch_begin(0, 0); + String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + for EachNode(dn, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(scratch.arena); + CV_TypeIndexInfoList l = cv_get_inlinee_type_index_offsets(temp.arena, dn->string); + lnk_gc_visit_offsets(g, dn->string, l); + temp_end(temp); + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + lnk_gc_visit_offsets(g, leaf.data, l); + temp_end(temp); + } + scratch_end(scratch); +} + +typedef struct LNK_GCRingTask +{ + U8 **leaf_v; + Rng1U64 *ranges; + U64 *counts; + U64 *offsets; + LNK_GCNamePair *pairs; +} LNK_GCRingTask; + +// parallel: count UDT leaves with a unique_name per range (pass 0) / emit (hash,idx) pairs (pass 1) +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_count_task) +{ + LNK_GCRingTask *t = raw_task; + U64 n = 0; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { n += 1; } + } + } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) +{ + LNK_GCRingTask *t = raw_task; + U64 cur = t->offsets[task_id]; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { + U64 h = 14695981039346656037ull; + for EachIndex(c, ui.unique_name.size) { h = (h ^ ui.unique_name.str[c]) * 0x100000001b3ull; } + t->pairs[cur].hash = h; t->pairs[cur].idx = (U32)i; cur += 1; + } + } + } +} + +// one bulk-synchronous round of transitive closure: visit each marked-but-unexpanded leaf in +// this source, mark the leaves it references (and its unique_name UDT counterparts). Parallel; +// `expanded` is grabbed atomically so each leaf is visited once, `changed` flags progress. +internal +THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) +{ + LNK_GCTypes *g = raw_task; + CV_TypeIndexSource s = g->cur_source; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + if (!g->mark[s][i]) { continue; } + if (ins_atomic_u8_eval_assign(&g->expanded[s][i], 1)) { continue; } // already visited + + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + for EachNode(n, CV_TypeIndexInfo, l.first) { + CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); + U64 lo = g->min[n->source]; + if (ti >= lo) { + U64 ci = ti - lo; + if (ci < g->orig_n[n->source] && !g->mark[n->source][ci]) { g->mark[n->source][ci] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + } + } + temp_end(temp); + + if (s == CV_TypeIndexSource_TPI) { + for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { + if (!g->mark[CV_TypeIndexSource_TPI][j]) { g->mark[CV_TypeIndexSource_TPI][j] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + } + } + } + scratch_end(scratch); +} + +internal void +lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + + LNK_GCTypes g = {0}; + g.cv = cv; + U64 total_leaves = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.min[s] = types->min_type_indices[s]; + g.orig_n[s] = types->count[s]; + g.leaf_v[s] = types->v[s]; + g.mark[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); // zeroed + total_leaves += g.orig_n[s]; + } + + // mark roots: every type index referenced by a surviving symbol / inlinee record + g.do_rewrite = 0; + g.sym_ranges = tp_divide_work(scratch.arena, cv->symbol_input_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + + // link UDT leaves that share a unique_name into rings, so that marking any one (e.g. a + // forward ref reached as a member-pointer target) also keeps its full definition -- needed + // for the debugger to complete types referenced only by name. TPI only (IPI has no UDTs). + U64 n_tpi = g.orig_n[CV_TypeIndexSource_TPI]; + g.udt_next = push_array_no_zero(scratch.arena, U32, n_tpi ? n_tpi : 1); + for EachIndex(i, n_tpi) { g.udt_next[i] = (U32)i; } + { + LNK_GCRingTask rt = {0}; + rt.leaf_v = g.leaf_v[CV_TypeIndexSource_TPI]; + rt.ranges = tp_divide_work(scratch.arena, n_tpi, tp->worker_count); + rt.counts = push_array(scratch.arena, U64, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_count_task, &rt); + rt.offsets = offsets_from_counts_array_u64(scratch.arena, rt.counts, tp->worker_count); + U64 np = sum_array_u64(tp->worker_count, rt.counts); + rt.pairs = push_array_no_zero(scratch.arena, LNK_GCNamePair, np ? np : 1); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_fill_task, &rt); + + radsort(rt.pairs, np, lnk_gc_name_pair_is_before); + for (U64 a = 0; a < np; ) { + U64 b = a + 1; + while (b < np && rt.pairs[b].hash == rt.pairs[a].hash) { b += 1; } + for (U64 k = a; k < b; k += 1) { g.udt_next[rt.pairs[k].idx] = rt.pairs[(k + 1 < b) ? (k + 1) : a].idx; } + a = b; + } + } + + // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each + // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new + U32 changed = 0; + g.changed = &changed; + Rng1U64 *expand_ranges[CV_TypeIndexSource_COUNT]; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.expanded[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); + expand_ranges[s] = tp_divide_work(scratch.arena, g.orig_n[s], tp->worker_count); + } + do { + changed = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_ranges = expand_ranges[s]; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + } + } while (changed); + + // compact each source: assign new contiguous type indices to the kept leaves. The leaf + // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never + // passes the read cursor) and remap lives in scratch -- so the GC adds nothing to the arena + // that survives into the (peak) PDB build. + U64 kept_total = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.remap[s] = push_array_no_zero(scratch.arena, CV_TypeIndex, g.orig_n[s] ? g.orig_n[s] : 1); + U8 **v = g.leaf_v[s]; + U64 new_n = 0; + for EachIndex(idx, g.orig_n[s]) { + if (g.mark[s][idx]) { g.remap[s][idx] = (CV_TypeIndex)(g.min[s] + new_n); v[new_n++] = v[idx]; } + else { g.remap[s][idx] = 0; /* T_NOTYPE; never referenced by a kept record */ } + } + types->count[s] = new_n; + kept_total += new_n; + } + + // rewrite all type-index references to the compacted indices + g.do_rewrite = 1; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_leaf_v = types->v[s]; + g.cur_ranges = tp_divide_work(scratch.arena, types->count[s], tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_rewrite_leaves_task, &g); + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "type GC: kept %llu of %llu leaves (pruned %llu)", kept_total, total_leaves, total_leaves - kept_total); + } + + scratch_end(scratch); + ProfEnd(); +} + internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags) { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index aa49eee42..ee0e89e27 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -267,4 +267,5 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP //////////////////////////////// // PDB +internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags); From 61797ba59949f59b15faf003d3660140cc049bd0 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 09:31:30 -0700 Subject: [PATCH 17/27] radlink: /OPT:ICF identical COMDAT folding (code + read-only data), parallelized Fold byte-identical COMDAT sections whose relocations point at equivalent targets, iterated to a fixpoint, then redirect each group's followers at their shared symbol-table node so every reference resolves to one leader and /OPT:REF collects the now-unreferenced follower sections (and their associated .pdata/.xdata/.debug$S). Mirrors link.exe /OPT:ICF. - equivalence: round-0 key from content + reloc structure + non-candidate target identity; refine by candidate targets' colors until the partition is stable; a final byte-compare + per-reloc target-color check guards folds (no hash-collision can produce a bad fold) - folds code AND read-only data (vtables, const tables, string literals); folding identical read-only data lets the functions that reference it fold too (cascade) - fully parallel: candidate collection (count -> exact alloc -> fill), content hashing + reloc-target resolution, refinement, and final grouping via a parallel LSD radix sort (8-bit digits). A flat open-addressing map with an avalanche-scrambled key avoids O(n^2) probing on UE-scale inputs. Only externally-defined COMDATs are folded (the follower redirects through its symbol). Static/internal-linkage folding is intentionally out of scope here. Numbers (UnrealEditorFortnite-Engine.dll, vs /OPT:NOICF, hashing NONE): .text 727 -> 643 MiB (-84) .rdata 218 -> 194 MiB (-24) PDB 5562 -> 5081 MB (-481) DLL 999 -> 882 MB (-117) link 33 -> 20 s (-13; less to relocate and emit downstream) This commit also adds the shared parallel radix-sort helper (lnk_radix_sort_u64_pairs) used here and by the PDB GSI/PSI sort. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 495 ++++++++++++++++++++++++++++++++++++++++++++++- src/linker/lnk.h | 1 + 2 files changed, 495 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 0eafb8439..ad1c7f69d 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -74,6 +74,8 @@ #include "pdb_ext/msf_builder.c" #include "pdb_ext/pdb.c" #include "pdb_ext/pdb_helpers.c" +// fwd decl: parallel radix sort (defined later in this TU) so pdb_builder.c can use it +internal void lnk_radix_sort_u64_pairs(TP_Context *tp, Arena *arena, U64 n, U64 *keys, U32 *vals); #include "pdb_ext/pdb_builder.c" // --- RDI --------------------------------------------------------------------- @@ -2377,6 +2379,13 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer ProfEnd(); } + // + // fold identical code COMDATs (routes followers to a leader; /OPT:REF then GCs them) + // + if (config->opt_icf == LNK_SwitchState_Yes) { + lnk_opt_icf(tp, symtab, config, link->objs); + } + // // discard COMDAT sections that are not referenced // @@ -2658,7 +2667,7 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) if (section_flags & LNK_SECTION_FLAG_DEBUG) { continue; } LNK_RelocRefs refs = {0}; - refs.obj = ref_symbol.obj; + refs.obj = ref_symbol.obj; refs.relocs = lnk_coff_reloc_info_from_section_number(ref_symbol.obj, section_number); // get a batch node @@ -2765,6 +2774,485 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) ProfEnd(); } +internal U64 +lnk_icf_mix(U64 h, U64 x) +{ + h ^= x; + h *= 0x100000001b3ull; + h ^= h >> 29; + return h; +} + +// Flat open-addressing U64 -> U64 map used by /OPT:ICF. The tree-based HashMap is far too slow +// for the millions of per-round dense-id lookups; this linear-probing table keeps the hot loops +// cache-friendly. Empty slots hold key == LNK_ICF_EMPTY (a value the stored hashes never take). +#define LNK_ICF_EMPTY 0xffffffffffffffffull +typedef struct LNK_ICFMap +{ + U64 *keys; + U64 *vals; + U64 mask; +} LNK_ICFMap; + +// avalanche so structured keys (e.g. (input_idx<<32)|section_number, whose low bits repeat +// across objects) scatter across slots instead of clustering into long probe chains +internal U64 +lnk_icf_scramble(U64 x) +{ + x ^= x >> 33; + x *= 0xff51afd7ed558ccdull; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ull; + x ^= x >> 33; + return x; +} + +internal LNK_ICFMap +lnk_icf_map_make(Arena *arena, U64 capacity) +{ + U64 cap = 1; + while (cap < capacity*2) { cap <<= 1; } + LNK_ICFMap m = {0}; + m.keys = push_array_no_zero(arena, U64, cap); + m.vals = push_array_no_zero(arena, U64, cap); + m.mask = cap - 1; + for EachIndex(i, cap) { m.keys[i] = LNK_ICF_EMPTY; } + return m; +} + +// look up key; returns stored value or fallback if absent +internal U64 +lnk_icf_map_get(LNK_ICFMap *m, U64 key, U64 fallback) +{ + U64 slot = lnk_icf_scramble(key) & m->mask; + for (;;) { + U64 k = m->keys[slot]; + if (k == key) { return m->vals[slot]; } + if (k == LNK_ICF_EMPTY) { return fallback; } + slot = (slot + 1) & m->mask; + } +} + +internal void +lnk_icf_map_put(LNK_ICFMap *m, U64 key, U64 val) +{ + U64 slot = lnk_icf_scramble(key) & m->mask; + for (;;) { + U64 k = m->keys[slot]; + if (k == LNK_ICF_EMPTY || k == key) { m->keys[slot] = key; m->vals[slot] = val; return; } + slot = (slot + 1) & m->mask; + } +} + +typedef struct LNK_ICFCand +{ + LNK_Obj *obj; + U32 sn; // section number + U32 reloc_first; // index into flattened reloc-target arrays + U32 reloc_count; + U64 key0; // round-0 content key + U64 color; // current equivalence class +} LNK_ICFCand; + +typedef struct LNK_ICFHashTask +{ + Rng1U64 *ranges; + LNK_ICFCand *cands; + LNK_ICFMap *cand_map; + LNK_SymbolTable *symtab; + U8 *rt_iscand; + U64 *rt_target; +} LNK_ICFHashTask; + +// per-candidate content hash + relocation-target resolution (parallel; each candidate writes +// its own disjoint reloc slice and key0, all reads are of immutable structures) +internal +THREAD_POOL_TASK_FUNC(lnk_icf_hash_task) +{ + LNK_ICFHashTask *task = raw_task; + for EachInRange(ci, task->ranges[task_id]) { + LNK_ICFCand *c = &task->cands[ci]; + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(c->obj, c->sn); + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(c->obj, header); + String8 data = str8_substr(c->obj->data, rng_1u64(header->foff, header->foff + header->fsize)); + + blake3_hasher h; blake3_hasher_init(&h); + U32 flags_for_hash = c->obj->section_flags[c->sn - 1] & ~(COFF_SectionFlag_LnkCOMDAT | COFF_SectionFlag_LnkRemove); + blake3_hasher_update(&h, &flags_for_hash, sizeof(flags_for_hash)); + blake3_hasher_update(&h, &header->fsize, sizeof(header->fsize)); + blake3_hasher_update(&h, data.str, data.size); + + for EachIndex(ri, relocs.count) { + COFF_Reloc *reloc = &relocs.v[ri]; + blake3_hasher_update(&h, &reloc->type, sizeof(reloc->type)); + blake3_hasher_update(&h, &reloc->apply_off, sizeof(reloc->apply_off)); + + COFF_ParsedSymbol tp = lnk_parsed_symbol_from_coff_symbol_idx(c->obj, reloc->isymbol); + COFF_SymbolValueInterpType ti = coff_interp_from_parsed_symbol(tp); + LNK_ObjSymbolRef tref = { c->obj, reloc->isymbol }; + if (ti == COFF_SymbolValueInterp_Undefined || ti == COFF_SymbolValueInterp_Weak) { + LNK_ObjSymbolRef resolved = {0}; + if (lnk_resolve_symbol(task->symtab, tref, &resolved)) { + tref = resolved; + tp = lnk_parsed_symbol_from_coff_symbol_idx(tref.obj, tref.symbol_idx); + ti = coff_interp_from_parsed_symbol(tp); + } + } + + U8 iscand = 0; + U64 target = 0; + if (ti == COFF_SymbolValueInterp_Regular && tref.obj != 0) { + U64 cv = lnk_icf_map_get(task->cand_map, Compose64Bit(tref.obj->input_idx, tp.section_number), 0); + if (cv) { iscand = 1; target = cv - 1; } + else { target = lnk_icf_mix(Compose64Bit(tref.obj->input_idx, tp.section_number), tp.value); } + } else { + U64 nh = 14695981039346656037ull; + for (U64 i = 0; i < tp.name.size; i += 1) { nh = lnk_icf_mix(nh, tp.name.str[i]); } + target = lnk_icf_mix(nh, tp.value); + } + + U64 idx = (U64)c->reloc_first + ri; + task->rt_iscand[idx] = iscand; + task->rt_target[idx] = target; + if (!iscand) { blake3_hasher_update(&h, &target, sizeof(target)); } + } + + U8 out[16]; blake3_hasher_finalize(&h, out, sizeof(out)); + U64 lo = *(U64 *)&out[0], hi = *(U64 *)&out[8]; + c->key0 = lnk_icf_mix(lo, hi); + } +} + +typedef struct LNK_ICFRefineTask +{ + Rng1U64 *ranges; + LNK_ICFCand *cands; + U8 *rt_iscand; + U64 *rt_target; + U64 *newkey; +} LNK_ICFRefineTask; + +// recompute each candidate's refinement key from its current color and its targets' colors +// (parallel; reads the immutable color snapshot, writes its own newkey slot) +internal +THREAD_POOL_TASK_FUNC(lnk_icf_refine_task) +{ + LNK_ICFRefineTask *task = raw_task; + for EachInRange(ci, task->ranges[task_id]) { + LNK_ICFCand *c = &task->cands[ci]; + U64 k = lnk_icf_mix(0x9e3779b97f4a7c15ull, c->color); + for EachIndex(j, c->reloc_count) { + U64 idx = (U64)c->reloc_first + j; + U64 t = task->rt_iscand[idx] ? task->cands[task->rt_target[idx]].color : task->rt_target[idx]; + k = lnk_icf_mix(k, t); + } + task->newkey[ci] = k; + } +} + +// Is an object section an ICF fold candidate? Only externally-defined COMDATs are folded: the +// follower is redirected at its shared symbol-table node, so every reference (including from other +// objects) resolves to the leader and /OPT:REF then drops the follower. Returns 1 = candidate. +internal U32 +lnk_icf_section_kind(LNK_Obj *obj, U64 sect_idx) +{ + COFF_SectionFlags flags = obj->section_flags[sect_idx]; + if (~flags & COFF_SectionFlag_LnkCOMDAT) { return 0; } + if ( flags & COFF_SectionFlag_LnkRemove) { return 0; } + // fold code and read-only initialized data (const tables, vtables, string literals); folding + // identical read-only data lets functions that reference it fold too (cascade). Mutable data + // is never folded. + B32 is_code = (flags & COFF_SectionFlag_CntCode) != 0; + B32 is_rodata = (flags & COFF_SectionFlag_CntInitializedData) && !(flags & COFF_SectionFlag_MemWrite); + if (!is_code && !is_rodata) { return 0; } + U32 sn = (U32)sect_idx + 1; + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(obj, sn); + if (header->fsize == 0) { return 0; } + LNK_Symbol *sym = lnk_obj_get_comdat_symlink(obj, sn); + if (sym == 0) { return 0; } // no external defining symbol (static / internal-linkage) + LNK_ObjSymbolRef sym_ref = lnk_ref_from_symbol(sym); + if (sym_ref.obj != obj) { return 0; } // same-name follower (already removed) + COFF_ParsedSymbol sym_parsed = lnk_parsed_from_symbol(sym); + if (sym_parsed.section_number != sn || sym_parsed.value != 0) { return 0; } + return 1; // external leader +} + +typedef struct LNK_ICFCollectTask +{ + LNK_Obj **objs; + U64 *counts; + U64 *offsets; + LNK_ICFCand *cands; +} LNK_ICFCollectTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_icf_count_task) +{ + LNK_ICFCollectTask *t = raw_task; + LNK_Obj *obj = t->objs[task_id]; + U64 n = 0; + for EachIndex(si, obj->header.section_count_no_null) { if (lnk_icf_section_kind(obj, si)) { n += 1; } } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_icf_fill_task) +{ + LNK_ICFCollectTask *t = raw_task; + LNK_Obj *obj = t->objs[task_id]; + U64 cur = t->offsets[task_id]; + for EachIndex(si, obj->header.section_count_no_null) { + if (lnk_icf_section_kind(obj, si)) { + LNK_ICFCand *c = &t->cands[cur++]; + c->obj = obj; c->sn = (U32)si + 1; + c->reloc_first = 0; c->reloc_count = 0; c->key0 = 0; c->color = 0; + } + } +} + +// --- parallel LSD radix sort (U64 key + U32 value), 8 x 8-bit passes ------------------------- +// 8-bit digits keep the serial per-pass offset prefix tiny (256*workers, not 65536*workers); +// the extra passes are parallel histogram+scatter, so they cost little. +#define LNK_RADIX_BITS 8 +#define LNK_RADIX_SIZE (1 << LNK_RADIX_BITS) +typedef struct LNK_RadixSortTask +{ + Rng1U64 *ranges; + U64 *ksrc; U32 *vsrc; // read + U64 *kdst; U32 *vdst; // write (scatter) + U32 *hist; // [worker_count * LNK_RADIX_SIZE], per-worker digit offsets + U64 shift; +} LNK_RadixSortTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_radix_hist_task) +{ + LNK_RadixSortTask *t = raw_task; + U32 *h = t->hist + (U64)task_id * LNK_RADIX_SIZE; + for EachInRange(i, t->ranges[task_id]) { h[(t->ksrc[i] >> t->shift) & (LNK_RADIX_SIZE - 1)] += 1; } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_radix_scatter_task) +{ + LNK_RadixSortTask *t = raw_task; + U32 *h = t->hist + (U64)task_id * LNK_RADIX_SIZE; + for EachInRange(i, t->ranges[task_id]) { + U64 d = (t->ksrc[i] >> t->shift) & (LNK_RADIX_SIZE - 1); + U32 pos = h[d]++; + t->kdst[pos] = t->ksrc[i]; + t->vdst[pos] = t->vsrc[i]; + } +} + +// sort (keys[], vals[]) ascending by key, in place. arena supplies scratch (double buffers + histograms). +internal void +lnk_radix_sort_u64_pairs(TP_Context *tp, Arena *arena, U64 n, U64 *keys, U32 *vals) +{ + if (n < 2) { return; } + U64 W = tp->worker_count; + + Temp scratch = scratch_begin(&arena, 1); // internal buffers freed on return + LNK_RadixSortTask t = {0}; + t.ranges = tp_divide_work(scratch.arena, n, W); + t.hist = push_array_no_zero(scratch.arena, U32, W * LNK_RADIX_SIZE); + U64 *kbuf = push_array_no_zero(scratch.arena, U64, n); + U32 *vbuf = push_array_no_zero(scratch.arena, U32, n); + + U64 *ksrc = keys, *kdst = kbuf; + U32 *vsrc = vals, *vdst = vbuf; + for (U64 pass = 0; pass < 64 / LNK_RADIX_BITS; pass += 1) { + t.shift = pass * LNK_RADIX_BITS; + t.ksrc = ksrc; t.vsrc = vsrc; t.kdst = kdst; t.vdst = vdst; + + MemoryZero(t.hist, sizeof(U32) * W * LNK_RADIX_SIZE); + tp_for_parallel(tp, 0, W, lnk_radix_hist_task, &t); + + // exclusive prefix across (bucket, worker) so each worker writes a disjoint contiguous run + U64 running = 0; + for EachIndex(bucket, LNK_RADIX_SIZE) { + for EachIndex(w, W) { + U32 *slot = &t.hist[w * LNK_RADIX_SIZE + bucket]; + U32 c = *slot; + *slot = (U32)running; + running += c; + } + } + + tp_for_parallel(tp, 0, W, lnk_radix_scatter_task, &t); + + U64 *kt = ksrc; ksrc = kdst; kdst = kt; + U32 *vt = vsrc; vsrc = vdst; vdst = vt; + } + // even pass count -> sorted data is back in the original keys/vals arrays + scratch_end(scratch); +} + +// assign each candidate a dense equivalence-class id from its key, via a parallel sort + a +// cheap sequential group scan. Returns the number of distinct classes (for convergence). +internal U64 +lnk_icf_dense_colors(TP_Context *tp, Arena *arena, U64 n, U64 *keys, LNK_ICFCand *cands) +{ + Temp t = temp_begin(arena); + U64 *sk = push_array_no_zero(t.arena, U64, n ? n : 1); + U32 *sv = push_array_no_zero(t.arena, U32, n ? n : 1); + for EachIndex(ci, n) { sk[ci] = keys[ci]; sv[ci] = (U32)ci; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, sk, sv); + U64 nc = 0; + for EachIndex(k, n) { + if (k == 0 || sk[k] != sk[k - 1]) { nc += 1; } + cands[sv[k]].color = nc - 1; + } + temp_end(t); + return nc; +} + +// relocations point at equivalent targets (iteratively), then folds each group's followers +// into a single leader by routing them through the existing COMDAT symlink machinery and +// letting /OPT:REF garbage-collect the now-unreferenced follower sections (and their +// associated .pdata/.xdata/.debug$S). Mirrors link.exe /OPT:ICF. +internal void +lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs_list) +{ + if (config->opt_icf != LNK_SwitchState_Yes) { return; } + + ProfBeginFunction(); + Temp scratch = scratch_begin(0, 0); + Arena *arena = scratch.arena; + + U64 objs_count = objs_list.count; + LNK_Obj **objs = lnk_array_from_obj_list(arena, objs_list); + + // collect candidate sections (parallel per obj): count, exact-size, then fill at per-obj + // offsets. Counting first avoids sizing the array to the total section count (which would be + // ~10x larger and waste ~1GB on UE-scale inputs). + LNK_ICFCollectTask col = {0}; + col.objs = objs; + col.counts = push_array(arena, U64, objs_count ? objs_count : 1); + tp_for_parallel(tp, 0, objs_count, lnk_icf_count_task, &col); + col.offsets = offsets_from_counts_array_u64(arena, col.counts, objs_count); + U64 cand_count = sum_array_u64(objs_count, col.counts); + LNK_ICFCand *cands = push_array_no_zero(arena, LNK_ICFCand, cand_count ? cand_count : 1); + col.cands = cands; + tp_for_parallel(tp, 0, objs_count, lnk_icf_fill_task, &col); + + if (cand_count < 2) { goto done; } + + // build target lookup (input_idx, section_number) -> cand_idx+1, sized to the candidate count + LNK_ICFMap cand_map = lnk_icf_map_make(arena, cand_count); + for EachIndex(ci, cand_count) { + lnk_icf_map_put(&cand_map, Compose64Bit(cands[ci].obj->input_idx, cands[ci].sn), ci + 1); + } + + // flatten relocation targets and compute content keys + // count relocs and assign each candidate a disjoint slice in the flattened arrays + U64 total_relocs = 0; + for EachIndex(ci, cand_count) { + LNK_ICFCand *c = &cands[ci]; + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(c->obj, c->sn); + U64 rcount = lnk_coff_relocs_from_section_header(c->obj, header).count; + c->reloc_first = (U32)total_relocs; + c->reloc_count = (U32)rcount; + total_relocs += rcount; + } + U8 *rt_iscand = push_array_no_zero(arena, U8, total_relocs ? total_relocs : 1); + U64 *rt_target = push_array_no_zero(arena, U64, total_relocs ? total_relocs : 1); + + { + LNK_ICFHashTask hash_task = {0}; + hash_task.ranges = tp_divide_work(arena, cand_count, tp->worker_count); + hash_task.cands = cands; + hash_task.cand_map = &cand_map; + hash_task.symtab = symtab; + hash_task.rt_iscand = rt_iscand; + hash_task.rt_target = rt_target; + tp_for_parallel(tp, 0, tp->worker_count, lnk_icf_hash_task, &hash_task); + } + + // assign initial colors from content key (parallel sort + group scan) + U64 *newkey = push_array_no_zero(arena, U64, cand_count ? cand_count : 1); + for EachIndex(ci, cand_count) { newkey[ci] = cands[ci].key0; } + U64 class_count = lnk_icf_dense_colors(tp, arena, cand_count, newkey, cands); + + // iteratively refine classes by relocation-target classes until stable + LNK_ICFRefineTask refine_task = {0}; + refine_task.ranges = tp_divide_work(arena, cand_count, tp->worker_count); + refine_task.cands = cands; + refine_task.rt_iscand = rt_iscand; + refine_task.rt_target = rt_target; + refine_task.newkey = newkey; + for (U64 round = 0; round < 30; round += 1) { + tp_for_parallel(tp, 0, tp->worker_count, lnk_icf_refine_task, &refine_task); + U64 new_class_count = lnk_icf_dense_colors(tp, arena, cand_count, newkey, cands); + if (new_class_count == class_count) { break; } + class_count = new_class_count; + } + + // group candidates by final color (parallel sort) and fold followers into a leader + U64 *keys = newkey; // reuse + U32 *sci = push_array_no_zero(arena, U32, cand_count ? cand_count : 1); + for EachIndex(ci, cand_count) { keys[ci] = cands[ci].color; sci[ci] = (U32)ci; } + lnk_radix_sort_u64_pairs(tp, arena, cand_count, keys, sci); + + U64 fold_count = 0; + for (U64 i = 0; i < cand_count; ) { + U64 j = i + 1; + U64 color = keys[i]; + while (j < cand_count && keys[j] == color) { j += 1; } + + if (j - i >= 2) { + // leader = lowest (input_idx, sn) member, for deterministic output + U64 leader_oi = i; + for (U64 k = i + 1; k < j; k += 1) { + LNK_ICFCand *a = &cands[sci[leader_oi]]; + LNK_ICFCand *b = &cands[sci[k]]; + if (Compose64Bit(b->obj->input_idx, b->sn) < Compose64Bit(a->obj->input_idx, a->sn)) { leader_oi = k; } + } + LNK_ICFCand *L = &cands[sci[leader_oi]]; + COFF_SectionHeader *Lheader = lnk_coff_section_header_from_section_number(L->obj, L->sn); + String8 Ldata = str8_substr(L->obj->data, rng_1u64(Lheader->foff, Lheader->foff + Lheader->fsize)); + LNK_SymbolHashTrie *Lnode = L->obj->symlinks[L->sn]; + + for (U64 k = i; k < j; k += 1) { + if (k == leader_oi) { continue; } + LNK_ICFCand *F = &cands[sci[k]]; + COFF_SectionHeader *Fheader = lnk_coff_section_header_from_section_number(F->obj, F->sn); + + // verify byte-identical content + reloc structure (guards against hash collisions) + if (Fheader->fsize != Lheader->fsize) { continue; } + if (F->reloc_count != L->reloc_count) { continue; } + String8 Fdata = str8_substr(F->obj->data, rng_1u64(Fheader->foff, Fheader->foff + Fheader->fsize)); + if (!str8_match(Ldata, Fdata, 0)) { continue; } + B32 relocs_match = 1; + for EachIndex(t, L->reloc_count) { + U64 li = L->reloc_first + t, fi = F->reloc_first + t; + U64 lt = rt_iscand[li] ? cands[rt_target[li]].color : rt_target[li]; + U64 ft = rt_iscand[fi] ? cands[rt_target[fi]].color : rt_target[fi]; + if (lt != ft) { relocs_match = 0; break; } + } + if (!relocs_match) { continue; } + + // redirect at the shared symbol-table node so every object resolving this name points at + // the leader's definition; /OPT:REF then removes the now-unreferenced follower section. + LNK_SymbolHashTrie *Fnode = F->obj->symlinks[F->sn]; + if (Fnode && Fnode != Lnode) { + Fnode->symbol = Lnode->symbol; + fold_count += 1; + } + } + } + i = j; + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "/OPT:ICF folded %llu of %llu code COMDATs into %llu classes", fold_count, cand_count, class_count); + } + + done:; + scratch_end(scratch); + ProfEnd(); +} + internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs) { @@ -5428,6 +5916,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + // prune merged types not reachable from any surviving symbol (transparent PDB-size win) + if (config->opt_ref == LNK_SwitchState_Yes) { + lnk_gc_types(tp, arena->v[0], &cv, &cv_types); + } + // // Debug Info // diff --git a/src/linker/lnk.h b/src/linker/lnk.h index d6388d0a0..b442a9b61 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -391,6 +391,7 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs); +internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs); // --- Win32 Image ------------------------------------------------------------- From 252aed48adece36c9fa1b9a39b5bb9127a683132 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 09:31:31 -0700 Subject: [PATCH 18/27] radlink/pdb: coalesce DBI section contributions; stabilize + parallelize GSI/PSI sort DBI section contributions: - after sorting, merge contiguous contributions that share (section, module, flags), absorbing the alignment-padding gaps between them. On UE-scale input this collapses ~12.5M contribution records to ~2.0M and shrinks the DBI stream 367 -> 72 MB, with no change to the address map. GSI/PSI publics sort: - the comparators got element-stable tiebreakers (sort by record offset / dereferenced symbol identity, not by slot pointer) so the median-of-9 quicksort cannot degrade to O(n^2) on the large runs of equal-address / equal-name records that ICF now produces. - gsi_record_sort_by_sc returns a radix-sorted permutation index (via the shared lnk_radix_sort_u64_pairs) and the PSI address map is built from it, replacing a comparator sort that stalled multiple seconds on ICF-heavy links. Numbers (UnrealEditorFortnite-Engine.dll): DBI stream 367 -> 72 MB removes a multi-second GSI/PSI sort stall on ICF-folded inputs Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/pdb_ext/pdb_builder.c | 67 ++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 0c6e58095..ba7d5ace9 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1783,7 +1783,11 @@ psi_addr_map_compar_is_before(void *raw_a, void *raw_b) } else if (a->isect_off.off != b->isect_off.off) { is_before = a->isect_off.off < b->isect_off.off; } else { - is_before = str8_compar_case_sensitive(&a->name, &b->name) < 0; + int cmp = str8_compar_case_sensitive(&a->name, &b->name); + // unique, element-stable tiebreaker (offset is unique per record and travels with the + // element across radsort swaps) so equal-key runs (many ICF-folded symbols at one + // address) do not degrade radsort's quicksort to O(n^2) + is_before = cmp != 0 ? (cmp < 0) : (a->offset < b->offset); } return is_before; @@ -1797,12 +1801,22 @@ gsi_record_sort_by_name(PDB_GsiSortRecord *arr, U64 count) ProfEnd(); } -internal void -gsi_record_sort_by_sc(PDB_GsiSortRecord *arr, U64 count) +// returns a record-index permutation sorted ascending by (isect, off) -- the PSI address map +// order. A stable parallel radix sort on the U64 (isect<<32|off) key; equal addresses (e.g. +// ICF-folded symbols) keep input order, which is fine for an address->symbol map. +internal U32 * +gsi_record_sort_by_sc(TP_Context *tp, Arena *arena, PDB_GsiSortRecord *arr, U64 count) { ProfBeginFunction(); - radsort(arr, count, psi_addr_map_compar_is_before); + U64 *keys = push_array_no_zero(arena, U64, count ? count : 1); + U32 *idx = push_array_no_zero(arena, U32, count ? count : 1); + for EachIndex(i, count) { + keys[i] = ((U64)arr[i].isect_off.isect << 32) | (U64)arr[i].isect_off.off; + idx[i] = (U32)i; + } + lnk_radix_sort_u64_pairs(tp, arena, count, keys, idx); ProfEnd(); + return idx; } internal @@ -1839,7 +1853,9 @@ gsi_symbol_is_before(void *raw_a, void *raw_b) } } } - is_before = cmp < 0; + // unique, element-stable tiebreaker (symbol pointer travels with the element across + // radsort swaps) prevents O(n^2) radsort on equal-key runs (ICF-folded symbols) + is_before = cmp != 0 ? (cmp < 0) : ((U64)a < (U64)b); } return is_before; @@ -1871,7 +1887,9 @@ gsi_pub_symbol_is_before(void *raw_a, void *raw_b) } } - is_before = cmp < 0; + // unique, element-stable tiebreaker (symbol pointer travels with the element across + // radsort swaps) prevents O(n^2) radsort on equal-key runs (ICF-folded publics) + is_before = cmp != 0 ? (cmp < 0) : ((U64)a < (U64)b); } return is_before; @@ -2212,15 +2230,15 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe ProfBegin("Address Map"); ProfBegin("Sort"); - gsi_record_sort_by_sc(gsi_build.sort_record_arr, gsi_build.hash_record_count); + U32 *sorted = gsi_record_sort_by_sc(tp, scratch.arena, gsi_build.sort_record_arr, gsi_build.hash_record_count); ProfEnd(); - + ProfBegin("Offset Fill"); U64 addr_map_count = gsi_build.hash_record_count; U64 addr_map_size = addr_map_count * sizeof(U32); U32 *addr_map = push_array_no_zero(scratch.arena, U32, addr_map_count); for (U64 i = 0; i < addr_map_count; i += 1) { - addr_map[i] = gsi_build.sort_record_arr[i].offset; + addr_map[i] = gsi_build.sort_record_arr[sorted[i]].offset; } ProfEnd(); @@ -2775,12 +2793,39 @@ dbi_build_sec_con(Arena *arena, PDB_DbiContext *dbi) // sort section contribs so they are binary searchable lnk_radix_sort_dbi_sc_array(sc_array, dbi->sec_contrib_list.count, dbi->section_list.count + 1); - + + // coalesce adjacent contributions that belong to the same module, section, and + // flags and are perfectly contiguous. The section contribution substream is only + // used to map an image address range back to its module, so merging contiguous + // same-module runs is loss-less (data/reloc CRCs are unused here, always 0). UE + // objects emit one COMDAT section per function, so after layout these collapse + // heavily -- matching link.exe, which stores one entry per contiguous run. + ProfBegin("Coalesce sect contribs"); + U64 sc_count = 0; + for (U64 r = 0; r < dbi->sec_contrib_list.count; r += 1) { + PDB_DbiSectionContrib *c = &sc_array[r]; + if (sc_count > 0) { + PDB_DbiSectionContrib *p = &sc_array[sc_count - 1]; + if (p->base.sec == c->base.sec && + p->base.mod == c->base.mod && + p->base.flags == c->base.flags && + (U64)c->base.sec_off >= (U64)p->base.sec_off) { + // absorb c into the run, extending over any alignment padding gap + U64 c_end = (U64)c->base.sec_off + (U64)c->base.size; + U64 p_end = (U64)p->base.sec_off + (U64)p->base.size; + if (c_end > p_end) { p->base.size = (U32)(c_end - (U64)p->base.sec_off); } + continue; + } + } + sc_array[sc_count++] = *c; + } + ProfEnd(); + // push section contrib info ProfBegin("List Push"); String8List sec_con_list = {0}; str8_list_push(arena, &sec_con_list, str8((U8*)version, sizeof(*version))); - str8_list_push(arena, &sec_con_list, str8((U8*)sc_array, sizeof(sc_array[0])*dbi->sec_contrib_list.count)); + str8_list_push(arena, &sec_con_list, str8((U8*)sc_array, sizeof(sc_array[0])*sc_count)); ProfEnd(); ProfEnd(); From bd0fc82c7a4b0891e1177af5d8a371bc3a520a7d Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:10:58 -0700 Subject: [PATCH 19/27] radlink: frontier worklist for type-GC transitive closure (perf) The closure re-scanned every merged type each round (O(rounds * total types)) to find marked-but-unexpanded leaves. In a full-link trace of UnrealEditorFortnite that round-rescan dominated the type-GC: lnk_gc_expand_task ~13.3 s of CPU. Replace it with a frontier worklist: the atomic mark now gates a single append per leaf, and each round expands only the slice newly marked by the previous round, so total work is O(reachable types) instead of O(rounds * total types). Drops the per-round `expanded` bitmap and full-array sweeps. Output is unchanged -- same reachable set, PDB byte-identical (5081 MB on the same input), and debugger-fidelity checks (addr->symbol 100%, core types resolve) match the pre-change build. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 69 ++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 5e42f4877..8f2f14271 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3105,17 +3105,20 @@ typedef struct LNK_GCTypes U64 min [CV_TypeIndexSource_COUNT]; // first type index per source U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) - U8 *expanded[CV_TypeIndexSource_COUNT]; // transitive-closure: already-visited bitmap CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays U32 *udt_next; // TPI fwdref<->definition unique_name ring - U32 *changed; // set when a closure round marks something new Rng1U64 *sym_ranges; B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // transitive-closure frontier: indices marked but not yet expanded. Each leaf is appended once + // (the atomic mark gates it), so frontier[s] is sized orig_n[s] and fcount[s] is its atomic tail. + U32 *frontier[CV_TypeIndexSource_COUNT]; + U32 *fcount [CV_TypeIndexSource_COUNT]; // atomic append cursor per source // per-source scratch (set before dispatch) CV_TypeIndexSource cur_source; U8 **cur_leaf_v; Rng1U64 *cur_ranges; + U64 round_begin, round_end; // frontier slice processed this round } LNK_GCTypes; typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; @@ -3243,18 +3246,29 @@ THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) } } -// one bulk-synchronous round of transitive closure: visit each marked-but-unexpanded leaf in -// this source, mark the leaves it references (and its unique_name UDT counterparts). Parallel; -// `expanded` is grabbed atomically so each leaf is visited once, `changed` flags progress. +// mark a leaf reachable and, if this is its first mark, append it to its source's frontier so a +// later round expands it. Atomic mark gates the append, so each leaf lands on the frontier once. +internal void +lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) +{ + if (ci < g->orig_n[ns] && !ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; + g->frontier[ns][pos] = (U32)ci; + } +} + +// one bulk-synchronous round of transitive closure: expand the frontier slice [round_begin, +// round_end) of cur_source -- visit each leaf and enqueue the leaves it references (and its +// unique_name UDT counterparts). Frontier-driven, so total work is O(reachable leaves), not +// O(rounds * total leaves). internal THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) { LNK_GCTypes *g = raw_task; CV_TypeIndexSource s = g->cur_source; Temp scratch = scratch_begin(0, 0); - for EachInRange(i, g->cur_ranges[task_id]) { - if (!g->mark[s][i]) { continue; } - if (ins_atomic_u8_eval_assign(&g->expanded[s][i], 1)) { continue; } // already visited + for EachInRange(local, g->cur_ranges[task_id]) { + U32 i = g->frontier[s][g->round_begin + local]; Temp temp = temp_begin(scratch.arena); CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); @@ -3262,16 +3276,13 @@ THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) for EachNode(n, CV_TypeIndexInfo, l.first) { CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); U64 lo = g->min[n->source]; - if (ti >= lo) { - U64 ci = ti - lo; - if (ci < g->orig_n[n->source] && !g->mark[n->source][ci]) { g->mark[n->source][ci] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } - } + if (ti >= lo) { lnk_gc_mark_enqueue(g, n->source, ti - lo); } } temp_end(temp); if (s == CV_TypeIndexSource_TPI) { for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { - if (!g->mark[CV_TypeIndexSource_TPI][j]) { g->mark[CV_TypeIndexSource_TPI][j] = 1; ins_atomic_u32_eval_assign(g->changed, 1); } + lnk_gc_mark_enqueue(g, CV_TypeIndexSource_TPI, j); } } } @@ -3329,21 +3340,31 @@ lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedType // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new - U32 changed = 0; - g.changed = &changed; - Rng1U64 *expand_ranges[CV_TypeIndexSource_COUNT]; + // seed the frontier with the root-marked leaves (one O(total leaves) scan), then expand + // frontier slices until both sources drain. Each leaf is expanded exactly once. + U32 fcount[CV_TypeIndexSource_COUNT] = {0}; + U64 start [CV_TypeIndexSource_COUNT] = {0}; for EachIndex(s, CV_TypeIndexSource_COUNT) { - g.expanded[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); - expand_ranges[s] = tp_divide_work(scratch.arena, g.orig_n[s], tp->worker_count); + g.frontier[s] = push_array_no_zero(scratch.arena, U32, g.orig_n[s] ? g.orig_n[s] : 1); + g.fcount[s] = &fcount[s]; + for EachIndex(i, g.orig_n[s]) { if (g.mark[s][i]) { g.frontier[s][fcount[s]++] = (U32)i; } } } - do { - changed = 0; + for (;;) { + B32 any = 0; for EachIndex(s, CV_TypeIndexSource_COUNT) { - g.cur_source = (CV_TypeIndexSource)s; - g.cur_ranges = expand_ranges[s]; - tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + U64 begin = start[s], end = fcount[s]; // fcount may grow during the round (cross-source enqueues) + if (begin < end) { + any = 1; + g.cur_source = (CV_TypeIndexSource)s; + g.round_begin = begin; + g.round_end = end; + g.cur_ranges = tp_divide_work(scratch.arena, end - begin, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + start[s] = end; + } } - } while (changed); + if (!any) { break; } + } // compact each source: assign new contiguous type indices to the kept leaves. The leaf // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never From 3bb86c2499e6821f05d2b2f8bd3ec9e38219fa2b Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:22:37 -0700 Subject: [PATCH 20/27] radlink: count ICF reloc slices in the parallel fill, not a serial re-parse lnk_opt_icf re-parsed every candidate section serially (lnk_coff_relocs_from_ section_header per candidate) just to size the flattened reloc-target arrays. Move the per-candidate reloc count into lnk_icf_fill_task -- which is already parallel and has the section in hand -- so lnk_opt_icf only does a cheap serial prefix sum for reloc_first. No output change (PDB/.text/.pdata identical). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index ad1c7f69d..350f88fc0 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3005,7 +3005,12 @@ THREAD_POOL_TASK_FUNC(lnk_icf_fill_task) if (lnk_icf_section_kind(obj, si)) { LNK_ICFCand *c = &t->cands[cur++]; c->obj = obj; c->sn = (U32)si + 1; - c->reloc_first = 0; c->reloc_count = 0; c->key0 = 0; c->color = 0; + // count relocs here (parallel, the section is already in hand) so lnk_opt_icf only needs a + // cheap serial prefix sum for reloc_first instead of re-parsing every section serially. + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(obj, c->sn); + c->reloc_first = 0; + c->reloc_count = (U32)lnk_coff_relocs_from_section_header(obj, header).count; + c->key0 = 0; c->color = 0; } } } @@ -3144,16 +3149,12 @@ lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj lnk_icf_map_put(&cand_map, Compose64Bit(cands[ci].obj->input_idx, cands[ci].sn), ci + 1); } - // flatten relocation targets and compute content keys - // count relocs and assign each candidate a disjoint slice in the flattened arrays + // assign each candidate a disjoint slice in the flattened reloc-target arrays. reloc_count was + // filled in parallel by lnk_icf_fill_task, so this is just a serial prefix sum (no re-parsing). U64 total_relocs = 0; for EachIndex(ci, cand_count) { - LNK_ICFCand *c = &cands[ci]; - COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(c->obj, c->sn); - U64 rcount = lnk_coff_relocs_from_section_header(c->obj, header).count; - c->reloc_first = (U32)total_relocs; - c->reloc_count = (U32)rcount; - total_relocs += rcount; + cands[ci].reloc_first = (U32)total_relocs; + total_relocs += cands[ci].reloc_count; } U8 *rt_iscand = push_array_no_zero(arena, U8, total_relocs ? total_relocs : 1); U64 *rt_target = push_array_no_zero(arena, U64, total_relocs ? total_relocs : 1); From f45f2003dbfc0ed6bdf70a92be375a24bb717992 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:28:17 -0700 Subject: [PATCH 21/27] radlink: skip the GC mark atomic on the already-reachable fast path The frontier mark did an interlocked op on every reference edge. Add a plain non-atomic check first (the mark bit only ever goes 0->1, so a stale "already set" read is safe), so the interlocked op runs once per leaf at its 0->1 transition instead of once per edge. Output identical (PDB 5081 MB). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk_debug_info.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 8f2f14271..c48e323bc 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -3251,7 +3251,11 @@ THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) internal void lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) { - if (ci < g->orig_n[ns] && !ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + if (ci >= g->orig_n[ns]) { return; } + if (g->mark[ns][ci]) { return; } // fast non-atomic skip: already reachable (the common edge) + // only the worker that wins the 0->1 transition appends, so the atomic runs once per leaf (not + // once per reference edge). + if (!ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; g->frontier[ns][pos] = (U32)ci; } From aceb5c35f06540a63b706193d031ab969ad2ec17 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 12:36:31 -0700 Subject: [PATCH 22/27] radlink: make type-GC opt-in (/OPT:GCTYPES), default off Type-GC prunes CodeView types not referenced by any surviving symbol. That is a PDB-size win, but it removes types a debugger can still legitimately cast to in the watch window (the reachable-from-symbols set is a subset of the castable-type set) -- which is why the same approach was reverted before after users reported losing the ability to cast in the watch. So gate it behind /OPT:GCTYPES, default OFF; only opt in when the smaller PDB is worth the reduced castable-type set. Numbers (UnrealEditorFortnite-Engine.dll): default PDB 5315 MB; with /OPT:GCTYPES 5081 MB (-234). LINK ok, no RelocationAgainstRemovedSection either way. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 6 ++++-- src/linker/lnk_config.c | 4 ++++ src/linker/lnk_config.h | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 350f88fc0..0034df388 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -5917,8 +5917,10 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); - // prune merged types not reachable from any surviving symbol (transparent PDB-size win) - if (config->opt_ref == LNK_SwitchState_Yes) { + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: + // it removes types that a debugger can still legitimately cast to in the watch window + // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. + if (config->opt_gc_types == LNK_SwitchState_Yes) { lnk_gc_types(tp, arena->v[0], &cv, &cv_types); } diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 9aa8caee0..58a24149c 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -1679,6 +1679,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List config->opt_lbr = LNK_SwitchState_Yes; } else if (str8_match_lit("nolibr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_No; + } else if (str8_match_lit("gctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_Yes; + } else if (str8_match_lit("nogctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_No; } else { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown option \"%S\"", param); } diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 0b6d79641..548b5cc1a 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -290,6 +290,7 @@ typedef struct LNK_Config LNK_SwitchState opt_ref; LNK_SwitchState opt_icf; LNK_SwitchState opt_lbr; + LNK_SwitchState opt_gc_types; // /OPT:GCTYPES -- prune unreferenced CodeView types. Default OFF: shrinks PDB but a pruned type can't be cast-to in the debugger watch window. U64 opt_iter_count; LNK_SwitchState import_table_emit_biat; LNK_SwitchState import_table_emit_uiat; From ab0c3e6c3e747e04acd253bded934edc6f79606e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 13:59:12 -0700 Subject: [PATCH 23/27] linker: make import-table generation deterministic Two data races in the parallel library search made radlink output nonreproducible: relinking the same inputs produced ~2.1M differing bytes (every reloc against an import could shift). Both originate in the generated DLL import objs (".idata"), whose symbol values (IAT slots, jump thunks) are laid out in the order the imports appear -- so any nondeterminism in the import set or order propagates to every call site that references an imported function. 1. Import member order. The parallel lib search appends discovered import members to link->imports in worker/round completion order, which is nondeterministic. Sort them into a stable total order (by link_symbol, member_idx tie-break) before generating the import objs. 2. Misindexed dedup flag. When a second reference to an already-queued import is found, lnk_queue_lib_member OR'd LinkedRegular/LinkedImp into import_member_infos[member_idx] -- but member_idx indexes the *currently searched* lib, not the import's lib. It must be is_queued_import->member_idx. The wrong (race-determined) slot got flagged, so whether an import emitted a jump thunk varied run to run, changing the import obj's symbol count. After both fixes, relinking UnrealEditorFortnite-Engine.dll is byte-identical except for the 20 bytes of intentional PE timestamp and PDB GUID/age (verified: 2,110,092 -> 20 differing bytes; output size unchanged). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 0034df388..b3fbbfa3e 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1250,6 +1250,20 @@ lnk_lib_member_ref_is_before(void *raw_a, void *raw_b) return lnk_symbol_is_before(g_sort_lib_member_context[(*a)->member_idx].link, g_sort_lib_member_context[(*b)->member_idx].link); } +internal int +lnk_import_ref_is_before(void *raw_a, void *raw_b) +{ + // total order over import members, independent of parallel lib-search discovery + // round so the generated import objs (and thus IAT/thunk symbol values) are + // deterministic. link_symbol is unique per queued import (dedup guarantees it); + // member_idx is a stable tie-break. + LNK_LibMemberRef **a = raw_a, **b = raw_b; + LNK_Symbol *sa = (*a)->link_symbol, *sb = (*b)->link_symbol; + if (lnk_symbol_is_before(sa, sb)) { return 1; } + if (lnk_symbol_is_before(sb, sa)) { return 0; } + return (*a)->member_idx < (*b)->member_idx; +} + internal LNK_LibMemberRef ** lnk_array_from_lib_member_list(Arena *arena, LNK_LibMemberRefList list) { @@ -1588,7 +1602,7 @@ lnk_queue_lib_member(Arena *arena, // do not queue second import member link -- flag member and continue U8 flag = str8_starts_with(link_symbol->name, str8_lit("__imp_")) ? LNK_LibMemberFlag_LinkedImp : LNK_LibMemberFlag_LinkedRegular; LNK_LibMemberInfo *import_member_infos = hash_map_search_raw_raw(&lib_member_info_hm, is_queued_import->lib); - ins_atomic_u8_or(&import_member_infos[member_idx].flags, flag); + ins_atomic_u8_or(&import_member_infos[is_queued_import->member_idx].flags, flag); } else { B32 do_queue; if (str8_starts_with(link_symbol->name, str8_lit("__imp_"))) { @@ -2006,7 +2020,15 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer String8List delayed_dll_names = {0}; String8List static_dll_names = {0}; - for EachNode(member_ref, LNK_LibMemberRef, link->imports.first) { + // sort import members into a deterministic total order before generating import + // objs: the parallel lib search appends to link->imports in nondeterministic + // discovery-round order, which would otherwise make the per-DLL import symbol + // layout (IAT slots, jump thunks) -- and every reloc against them -- nonreproducible. + LNK_LibMemberRef **import_refs = lnk_array_from_lib_member_list(scratch.arena, link->imports); + radsort(import_refs, link->imports.count, lnk_import_ref_is_before); + + for EachIndex(import_ref_idx, link->imports.count) { + LNK_LibMemberRef *member_ref = import_refs[import_ref_idx]; LNK_Lib *lib = member_ref->lib; U64 member_idx = member_ref->member_idx; LNK_LibMemberInfo *member_infos = hash_map_search_raw_raw(&link->lib_member_infos_hm, lib); From 8d00121b4e738579938f065c3a467150f1395ab8 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 10:49:33 -0700 Subject: [PATCH 24/27] radlink: parallelize the section-contrib sort for large chunks lnk_sort_contribs_task ran one serial radsort per chunk. A section's contribs live in a single chunk sized to the whole section, so the merged .text chunk (millions of entries) was sorted serially on one worker while every other thread idled -- the straggler that stretched the "Sort Section Contribs" phase. Sort chunks >= 64K entries with the parallel radix sort (key = Compose64Bit(obj_idx, obj_sect_idx), which is unique per contrib so the order matches the comparator) using all threads, before the per-chunk task pass handles the small remainder. Output is unchanged: section sizes identical, byte diff within the pre-existing relink noise. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index b3fbbfa3e..c43b8abc3 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -3486,11 +3486,33 @@ lnk_section_contrib_ptr_is_before(void *raw_a, void *raw_b) return u64_compar_is_before(&input_idx_a, &input_idx_b); } +// chunks at/above this size are sorted by the parallel radix (all threads) before the per-chunk +// task pass, so one giant section (e.g. merged .text) can't serialize the whole sort on one thread. +#define LNK_SORT_CONTRIBS_RADIX_MIN (64u*1024u) + +// sort one chunk's contribs by Compose64Bit(obj_idx, obj_sect_idx) using the parallel radix sort. +// The key is unique per contrib (one obj/section pair each), so this matches the radsort order. +internal void +lnk_sort_contribs_chunk_radix(TP_Context *tp, Arena *arena, LNK_SectionContribChunk *chunk) +{ + U64 n = chunk->count; + Temp t = temp_begin(arena); + U64 *keys = push_array_no_zero(t.arena, U64, n); + U32 *idx = push_array_no_zero(t.arena, U32, n); + for EachIndex(i, n) { keys[i] = Compose64Bit(chunk->v[i]->u.obj_idx, chunk->v[i]->u.obj_sect_idx); idx[i] = (U32)i; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, keys, idx); + LNK_SectionContrib **sorted = push_array_no_zero(t.arena, LNK_SectionContrib *, n); + for EachIndex(i, n) { sorted[i] = chunk->v[idx[i]]; } + MemoryCopy(chunk->v, sorted, n * sizeof(sorted[0])); + temp_end(t); +} + internal THREAD_POOL_TASK_FUNC(lnk_sort_contribs_task) { LNK_BuildImageTask *task = raw_task; LNK_SectionContribChunk *chunk = task->u.sort_contribs.chunks[task_id]; + if (chunk->count >= LNK_SORT_CONTRIBS_RADIX_MIN) { return; } // big chunks done via parallel radix ProfBeginV("[%llu]", chunk->count); radsort(chunk->v, chunk->count, lnk_section_contrib_ptr_is_before); ProfEnd(); @@ -4993,6 +5015,14 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT Assert(cursor == total_chunk_count); } + // big chunks (e.g. merged .text) first, each sorted across all threads via parallel radix -- + // otherwise a single huge chunk serializes on one worker. Small chunks then go one-per-task. + for EachIndex(ci, total_chunk_count) { + LNK_SectionContribChunk *chunk = task.u.sort_contribs.chunks[ci]; + if (chunk->count >= LNK_SORT_CONTRIBS_RADIX_MIN) { + lnk_sort_contribs_chunk_radix(tp, scratch.arena, chunk); + } + } tp_for_parallel(tp, 0, total_chunk_count, lnk_sort_contribs_task, &task); ProfEnd(); From c6903c2d20cf46766d00ffd0a9bd5bfe3a9bed58 Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 14:49:10 -0700 Subject: [PATCH 25/27] radlink: skip converged classes in ICF refinement ICF refinement re-densified all ~N candidates every round, radix-sorting the full set each iteration to a fixpoint. But a candidate alone in its equivalence class can never split or merge again -- its color is final. Track an active set of only the candidates still sharing a class with another, and re-densify just that set each round (ids drawn from an ever-increasing base so they never collide with the colors already finalized for singletons). The per-round sort shrinks from all candidates to those that still have a content+reloc twin, and converged classes drop out as they fragment into singletons. Output is unchanged -- relinking UnrealEditorFortnite-Engine.dll is byte-identical to the prior ICF (same folds, same size) and reproducible across runs. On that link the refinement loop drops from ~888ms to ~765ms (first round prunes ~2.1M of 3.98M candidates to singletons immediately). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 68 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index c43b8abc3..cefe09fd2 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -2952,6 +2952,7 @@ typedef struct LNK_ICFRefineTask U8 *rt_iscand; U64 *rt_target; U64 *newkey; + U32 *active; // when set, ranges index into active[] rather than cands[] directly } LNK_ICFRefineTask; // recompute each candidate's refinement key from its current color and its targets' colors @@ -2960,8 +2961,9 @@ internal THREAD_POOL_TASK_FUNC(lnk_icf_refine_task) { LNK_ICFRefineTask *task = raw_task; - for EachInRange(ci, task->ranges[task_id]) { - LNK_ICFCand *c = &task->cands[ci]; + for EachInRange(ai, task->ranges[task_id]) { + U64 ci = task->active ? task->active[ai] : ai; + LNK_ICFCand *c = &task->cands[ci]; U64 k = lnk_icf_mix(0x9e3779b97f4a7c15ull, c->color); for EachIndex(j, c->reloc_count) { U64 idx = (U64)c->reloc_first + j; @@ -2972,6 +2974,27 @@ THREAD_POOL_TASK_FUNC(lnk_icf_refine_task) } } +// densify the keys of just the active subset into class ids drawn from `base` upward (so they never +// collide with the finalized colors of candidates that already dropped out as singletons). Returns +// the number of distinct classes among the active set. Equal keys land in one class regardless of +// sort tie order, so color values are deterministic. +internal U64 +lnk_icf_dense_colors_active(TP_Context *tp, Arena *arena, U32 *active, U64 n, U64 *newkey, LNK_ICFCand *cands, U64 base) +{ + Temp t = temp_begin(arena); + U64 *sk = push_array_no_zero(t.arena, U64, n ? n : 1); + U32 *sv = push_array_no_zero(t.arena, U32, n ? n : 1); // index into active[] + for EachIndex(i, n) { sk[i] = newkey[active[i]]; sv[i] = (U32)i; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, sk, sv); + U64 nc = 0; + for EachIndex(k, n) { + if (k == 0 || sk[k] != sk[k - 1]) { nc += 1; } + cands[active[sv[k]]].color = base + (nc - 1); + } + temp_end(t); + return nc; +} + // Is an object section an ICF fold candidate? Only externally-defined COMDATs are folded: the // follower is redirected at its shared symbol-table node, so every reference (including from other // objects) resolves to the leader and /OPT:REF then drops the follower. Returns 1 = candidate. @@ -3196,19 +3219,48 @@ lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj U64 *newkey = push_array_no_zero(arena, U64, cand_count ? cand_count : 1); for EachIndex(ci, cand_count) { newkey[ci] = cands[ci].key0; } U64 class_count = lnk_icf_dense_colors(tp, arena, cand_count, newkey, cands); + U64 color_base = class_count; // next free class id; ids only ever grow, so finalized colors stick + + // Active set = candidates still sharing a class with another. A singleton class can never split + // or merge, so once a candidate is alone its color is final and it leaves refinement. Each round + // re-densifies only the active set, shrinking the per-round sort from all ~N candidates down to + // just those that still have a content+reloc twin. + U32 *active = push_array_no_zero(arena, U32, cand_count ? cand_count : 1); + U64 active_count = 0, active_class_count = 0; + { + Temp t = temp_begin(arena); + U32 *cls_size = push_array(t.arena, U32, class_count ? class_count : 1); + for EachIndex(ci, cand_count) { cls_size[cands[ci].color] += 1; } + for EachIndex(ci, cand_count) { if (cls_size[cands[ci].color] > 1) { active[active_count++] = (U32)ci; } } + for EachIndex(c, class_count) { if (cls_size[c] > 1) { active_class_count += 1; } } + temp_end(t); + } - // iteratively refine classes by relocation-target classes until stable + // iteratively refine classes by relocation-target classes until no class splits LNK_ICFRefineTask refine_task = {0}; - refine_task.ranges = tp_divide_work(arena, cand_count, tp->worker_count); refine_task.cands = cands; refine_task.rt_iscand = rt_iscand; refine_task.rt_target = rt_target; refine_task.newkey = newkey; - for (U64 round = 0; round < 30; round += 1) { + refine_task.active = active; + for (U64 round = 0; round < 30 && active_count > 0; round += 1) { + refine_task.ranges = tp_divide_work(arena, active_count, tp->worker_count); tp_for_parallel(tp, 0, tp->worker_count, lnk_icf_refine_task, &refine_task); - U64 new_class_count = lnk_icf_dense_colors(tp, arena, cand_count, newkey, cands); - if (new_class_count == class_count) { break; } - class_count = new_class_count; + + U64 base = color_base; + U64 nc = lnk_icf_dense_colors_active(tp, arena, active, active_count, newkey, cands, base); + color_base += nc; + if (nc == active_class_count) { break; } // no class split -> fixpoint + + // drop classes that just became singletons; keep the rest active for the next round + Temp t = temp_begin(arena); + U32 *cls_size = push_array(t.arena, U32, nc ? nc : 1); + for EachIndex(i, active_count) { cls_size[cands[active[i]].color - base] += 1; } + U64 na = 0, nac = 0; + for EachIndex(i, active_count) { if (cls_size[cands[active[i]].color - base] > 1) { active[na++] = active[i]; } } + for EachIndex(c, nc) { if (cls_size[c] > 1) { nac += 1; } } + active_count = na; active_class_count = nac; + temp_end(t); } // group candidates by final color (parallel sort) and fold followers into a leader From 351b2efba080a9b178f63d79af909ac80127fe1e Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 15:18:21 -0700 Subject: [PATCH 26/27] radlink: cache symbol interp so the lib search stops re-parsing resolved symbols lnk_search_lib_task scans the entire search-chunk symbol set once per library (2733 dispatches on the UE editor link). A symbol that started Undefined/Weak stays in search_chunks even after a definition resolves it, so every later library pass re-parsed it -- lnk_ref_from_symbol + lnk_parsed_symbol_from_coff _symbol_idx -- just to recompute its interp and skip it. That parse faults the COFF symbol record out of the mmap'd obj, and the profile showed those two lines at ~65% of the task and a matching wall of page-fault kernel time. The interp is already computed once in lnk_symbol_table_push_; cache it on LNK_Symbol and read it in the hot loop. Only genuinely Weak symbols still parse (for the weak-extension characteristics). The hash-trie node always points at the current leader symbol, so the cached interp reflects the resolved state. Output unchanged (byte-identical DLL+PDB, reproducible). Lib-search dispatch wall drops ~18% (~1.5s -> ~1.2s on the UE editor link) with a larger drop in aggregate CPU and page-fault traffic. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 10 ++++++---- src/linker/lnk_symbol_table.c | 1 + src/linker/lnk_symbol_table.h | 7 ++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index cefe09fd2..deeb7adf7 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1638,16 +1638,18 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) for EachIndex(i, c->count) { LNK_Symbol *symbol = c->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - // skip the name scan here; the resolved name is already cached on symbol->name - COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + // interp is cached on the symbol at push time, so the common case (resolved symbols, which + // stay in search_chunks but can never match a lib) costs one field read instead of re-parsing + // -- and page-faulting -- the COFF symbol record out of the mmap'd obj on every lib pass. + COFF_SymbolValueInterpType symbol_interp = symbol->interp; if (symbol_interp == COFF_SymbolValueInterp_Undefined) { U32 member_idx; if (lnk_search_lib(lib, symbol->name, &member_idx)) { lnk_queue_lib_member(arena, task->imports_hm, task->link->lib_member_infos_hm, member_ref_list, symbol, lib, lib_member_infos, member_idx); } } else if (symbol_interp == COFF_SymbolValueInterp_Weak) { + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(symbol_parsed, symbol_ref.obj->header.is_big_obj); if (weak_ext->characteristics == COFF_WeakExt_SearchLibrary) { U32 member_idx; diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index d1d117d08..470990520 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -562,6 +562,7 @@ lnk_symbol_table_push_(LNK_SymbolTable *symtab, Arena *arena, U64 worker_id, LNK { U64 hash = lnk_symbol_table_hasher(symbol->name); COFF_SymbolValueInterpType interp = lnk_interp_from_symbol(symbol); + symbol->interp = interp; // cache for the lib search hot loop LNK_SymbolHashTrieChunkList *chunks; if (interp == COFF_SymbolValueInterp_Weak || interp == COFF_SymbolValueInterp_Undefined) { chunks = &symtab->search_chunks[worker_id]; diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index b36a2ba64..fffd711c6 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -19,9 +19,10 @@ typedef struct LNK_ObjSymbolRefNode typedef struct LNK_Symbol { - String8 name; - LNK_ObjSymbolRefNode *refs; - LNK_ObjSymbolRefNode *refs_tail; // tail of `refs`, so symbol-ref merges are O(1) instead of O(n) tail walks + String8 name; + LNK_ObjSymbolRefNode *refs; + LNK_ObjSymbolRefNode *refs_tail; // tail of `refs`, so symbol-ref merges are O(1) instead of O(n) tail walks + COFF_SymbolValueInterpType interp; // cached at push so the lib search can skip re-parsing (and page-faulting) resolved symbols } LNK_Symbol; // --- Symbol Containers ------------------------------------------------------- From 9223d3004dce1be4fc6adcb68d6a122c7dd7b70c Mon Sep 17 00:00:00 2001 From: Henrik Karlsson Date: Fri, 19 Jun 2026 16:14:06 -0700 Subject: [PATCH 27/27] radlink: skip redundant library re-searches in the resolution fixpoint lnk_link_inputs resolves libraries to a fixpoint: an outer pass loops over every library, and each library is re-searched (a full tp_for_parallel over all workers, scanning every undefined/weak symbol in search_chunks) once per drained input batch until nothing new resolves. On the UE editor link that is ~2733 dispatches, each waking and joining ~60 workers -- and the phase is barrier-bound, so that wake/join is the cost, not the scan. Most re-searches are redundant: search_chunks only grows during the loop (symbols are never removed until the end) and member-queue dedup is idempotent, so a re-search can only queue new members if the undefined/weak symbol set grew or anti-dep searching was just enabled since this library was last searched. Stamp each LNK_Lib with the search_chunks symbol count + anti-dep mode at its last search and skip the dispatch when neither changed. ~24% fewer dispatches (2733 -> ~2089) and ~0.2s of wake/join wall-time removed. Output is byte-identical and reproducible (which dispatch coalesces is timing dependent, but a skipped one provably queued nothing, so the result is unchanged -- verified relink-twice byte-identical across many runs). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/linker/lnk.c | 36 +++++++++++++++++++++++++---------- src/linker/lnk_lib.h | 7 +++++++ src/linker/lnk_symbol_table.c | 15 +++++++++++++++ src/linker/lnk_symbol_table.h | 1 + 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/linker/lnk.c b/src/linker/lnk.c index deeb7adf7..07ba16624 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1813,16 +1813,32 @@ lnk_link_inputs(TP_Context *tp, } else { // search symbols in lib MemoryZeroTyped(member_ref_lists, tp->worker_count); - LNK_SearchLibTask search_task = { - .search_anti_deps = search_anti_deps, - .link = link, - .imports_hm = &imports_hm, - .lib = lib, - .symtab = symtab, - .lib_member_infos = lib_member_infos, - .member_ref_lists = member_ref_lists - }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); + + // barrier elision: the search task scans every undefined/weak symbol in search_chunks and + // queues any member that resolves one. search_chunks only grows during this loop and the + // dedup against already-queued members is idempotent, so if neither the symbol set nor the + // anti-dep mode changed since this lib was last searched, the dispatch can only re-queue + // (deduped to) nothing. skipping it avoids waking+joining every worker for no work. + U64 search_symbol_count = lnk_symbol_table_search_symbol_count(symtab); + B32 can_skip_search = lib->was_searched && + lib->searched_symbol_count == search_symbol_count && + lib->searched_anti_deps == search_anti_deps; + if ( ! can_skip_search) { + LNK_SearchLibTask search_task = { + .search_anti_deps = search_anti_deps, + .link = link, + .imports_hm = &imports_hm, + .lib = lib, + .symtab = symtab, + .lib_member_infos = lib_member_infos, + .member_ref_lists = member_ref_lists + }; + tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); + + lib->was_searched = 1; + lib->searched_symbol_count = search_symbol_count; + lib->searched_anti_deps = search_anti_deps; + } } LNK_LibMemberRefList queued_members = {0}; diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index eed5e2dea..9a939c163 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -15,6 +15,13 @@ typedef struct LNK_Lib String8Array symbol_names; String8 long_names; U64 input_idx; + + // lib-search barrier elision: a re-search of this lib can only queue new members if the set of + // undefined/weak symbols grew, or if anti-dep searching was just enabled, since the last search. + // these record the state at the last search so identical re-searches can skip the tp dispatch. + B32 was_searched; + B32 searched_anti_deps; + U64 searched_symbol_count; } LNK_Lib; typedef struct LNK_LibNode diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 470990520..cb02c3ce2 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -592,6 +592,21 @@ lnk_symbol_table_search(LNK_SymbolTable *symtab, String8 name) return trie ? trie->symbol : 0; } +internal U64 +lnk_symbol_table_search_symbol_count(LNK_SymbolTable *symtab) +{ + // total number of weak/undefined symbols inserted into search_chunks. this only ever grows + // during the lib-search loop (symbols are never removed mid-loop), so it serves as a monotonic + // version stamp for the set of symbols the lib search would scan. + U64 count = 0; + for EachIndex(worker_id, symtab->arena->count) { + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[worker_id].first) { + count += c->count; + } + } + return count; +} + internal LNK_Symbol * lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...) { diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index fffd711c6..4fbcfcf09 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -125,6 +125,7 @@ internal U64 lnk_symbol_table_hasher(String8 string); internal LNK_SymbolTable * lnk_symbol_table_init(TP_Arena *arena); internal void lnk_symbol_table_push(LNK_SymbolTable *symtab, LNK_Symbol *symbol); internal LNK_Symbol * lnk_symbol_table_search(LNK_SymbolTable *symtab, String8 name); +internal U64 lnk_symbol_table_search_symbol_count(LNK_SymbolTable *symtab); internal LNK_Symbol * lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...); // --- Symbol Contrib Helpers --------------------------------------------------