diff --git a/build.bat b/build.bat index 5c9c71d46..54f70b945 100644 --- a/build.bat +++ b/build.bat @@ -141,7 +141,14 @@ popd pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +:: NOTE: -DBLAKE3_ATOMICS=1 makes BLAKE3 use C11 _Atomic (plain atomic load) for +:: get_cpu_features instead of MSVC's _InterlockedOr `lock or` barrier on every +:: compress dispatch (was a ~5.5s main-thread hot spot). MSVC C11 atomics require +:: /std:c11 /experimental:c11atomics. Kept external so the vendored blake3 source +:: stays pristine. +set radlink_msvc_flags= +if "%msvc%"=="1" set radlink_msvc_flags=/std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1 +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_msvc_flags% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 diff --git a/src/base/base_strings.c b/src/base/base_strings.c index 8825d4798..78e3e59ac 100644 --- a/src/base/base_strings.c +++ b/src/base/base_strings.c @@ -213,7 +213,9 @@ str8_cstring_capped(void *cstr, void *cap) { char *ptr = (char *)cstr; char *opl = (char *)cap; - for (;ptr < opl && *ptr != 0; ptr += 1); + // memchr is typically SIMD-accelerated; much faster than a byte-by-byte scan + char *nul = (char *)memchr(ptr, 0, (U64)(opl - ptr)); + ptr = nul ? nul : opl; U64 size = (U64)(ptr - (char *)cstr); String8 result = str8((U8*)cstr, size); return result; diff --git a/src/base/base_threads.h b/src/base/base_threads.h index b34286ccb..9e156d593 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -129,6 +129,7 @@ internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); internal void semaphore_drop(Semaphore semaphore); +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/coff/coff_parse.c b/src/coff/coff_parse.c index 1db0b5826..85835b977 100644 --- a/src/coff/coff_parse.c +++ b/src/coff/coff_parse.c @@ -166,11 +166,16 @@ coff_section_header_array_from_name(Arena *arena, String8 string_table, COFF_Sec } +// NOTE: name-skipping variants. coff_read_symbol_name does a cstr scan over the +// memory-mapped string table, which is the dominant cost when parsing symbols in +// bulk; callers that only need the scalar fields (value/section/storage_class/aux, +// e.g. symbol-value interpretation) should use these to avoid that scan. The full +// coff_parse_symbol{16,32} below are these plus the name read, so the scalar-field +// logic lives in exactly one place. internal COFF_ParsedSymbol -coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +coff_parse_symbol32_no_name(COFF_Symbol32 *sym32) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym32->name); result.value = sym32->value; result.section_number = sym32->section_number; result.type = sym32->type; @@ -181,10 +186,9 @@ coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) } internal COFF_ParsedSymbol -coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +coff_parse_symbol16_no_name(COFF_Symbol16 *sym16) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym16->name); result.value = sym16->value; if (sym16->section_number == COFF_Symbol_DebugSection16) { result.section_number = COFF_Symbol_DebugSection32; @@ -200,6 +204,22 @@ coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) return result; } +internal COFF_ParsedSymbol +coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +{ + COFF_ParsedSymbol result = coff_parse_symbol32_no_name(sym32); + result.name = coff_read_symbol_name(string_table, &sym32->name); + return result; +} + +internal COFF_ParsedSymbol +coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +{ + COFF_ParsedSymbol result = coff_parse_symbol16_no_name(sym16); + result.name = coff_read_symbol_name(string_table, &sym16->name); + return result; +} + internal COFF_ParsedSymbol coff_parse_symbol(COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx) { diff --git a/src/coff/coff_parse.h b/src/coff/coff_parse.h index edcb29520..028e292f5 100644 --- a/src/coff/coff_parse.h +++ b/src/coff/coff_parse.h @@ -261,6 +261,8 @@ internal String8 coff_name_from_section_header (String8 str internal COFF_ParsedSymbol coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32); internal COFF_ParsedSymbol coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16); +internal COFF_ParsedSymbol coff_parse_symbol32_no_name(COFF_Symbol32 *sym32); +internal COFF_ParsedSymbol coff_parse_symbol16_no_name(COFF_Symbol16 *sym16); internal COFF_ParsedSymbol coff_parse_symbol (COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx); internal COFF_Symbol32Array coff_symbol_array_from_data_16(Arena *arena, String8 data, U64 symbol_array_off, U64 symbol_count); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 250a33c45..07ba16624 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -74,6 +74,8 @@ #include "pdb_ext/msf_builder.c" #include "pdb_ext/pdb.c" #include "pdb_ext/pdb_helpers.c" +// fwd decl: parallel radix sort (defined later in this TU) so pdb_builder.c can use it +internal void lnk_radix_sort_u64_pairs(TP_Context *tp, Arena *arena, U64 n, U64 *keys, U32 *vals); #include "pdb_ext/pdb_builder.c" // --- RDI --------------------------------------------------------------------- @@ -1248,6 +1250,20 @@ lnk_lib_member_ref_is_before(void *raw_a, void *raw_b) return lnk_symbol_is_before(g_sort_lib_member_context[(*a)->member_idx].link, g_sort_lib_member_context[(*b)->member_idx].link); } +internal int +lnk_import_ref_is_before(void *raw_a, void *raw_b) +{ + // total order over import members, independent of parallel lib-search discovery + // round so the generated import objs (and thus IAT/thunk symbol values) are + // deterministic. link_symbol is unique per queued import (dedup guarantees it); + // member_idx is a stable tie-break. + LNK_LibMemberRef **a = raw_a, **b = raw_b; + LNK_Symbol *sa = (*a)->link_symbol, *sb = (*b)->link_symbol; + if (lnk_symbol_is_before(sa, sb)) { return 1; } + if (lnk_symbol_is_before(sb, sa)) { return 0; } + return (*a)->member_idx < (*b)->member_idx; +} + internal LNK_LibMemberRef ** lnk_array_from_lib_member_list(Arena *arena, LNK_LibMemberRefList list) { @@ -1586,7 +1602,7 @@ lnk_queue_lib_member(Arena *arena, // do not queue second import member link -- flag member and continue U8 flag = str8_starts_with(link_symbol->name, str8_lit("__imp_")) ? LNK_LibMemberFlag_LinkedImp : LNK_LibMemberFlag_LinkedRegular; LNK_LibMemberInfo *import_member_infos = hash_map_search_raw_raw(&lib_member_info_hm, is_queued_import->lib); - ins_atomic_u8_or(&import_member_infos[member_idx].flags, flag); + ins_atomic_u8_or(&import_member_infos[is_queued_import->member_idx].flags, flag); } else { B32 do_queue; if (str8_starts_with(link_symbol->name, str8_lit("__imp_"))) { @@ -1622,15 +1638,18 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) for EachIndex(i, c->count) { LNK_Symbol *symbol = c->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + // interp is cached on the symbol at push time, so the common case (resolved symbols, which + // stay in search_chunks but can never match a lib) costs one field read instead of re-parsing + // -- and page-faulting -- the COFF symbol record out of the mmap'd obj on every lib pass. + COFF_SymbolValueInterpType symbol_interp = symbol->interp; if (symbol_interp == COFF_SymbolValueInterp_Undefined) { U32 member_idx; if (lnk_search_lib(lib, symbol->name, &member_idx)) { lnk_queue_lib_member(arena, task->imports_hm, task->link->lib_member_infos_hm, member_ref_list, symbol, lib, lib_member_infos, member_idx); } } else if (symbol_interp == COFF_SymbolValueInterp_Weak) { + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(symbol_parsed, symbol_ref.obj->header.is_big_obj); if (weak_ext->characteristics == COFF_WeakExt_SearchLibrary) { U32 member_idx; @@ -1641,11 +1660,11 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) if (search_anti_deps) { LNK_ObjSymbolRef dep_symbol = {0}; if (lnk_resolve_weak_symbol(symtab, symbol_ref, &dep_symbol)) { - COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx(dep_symbol.obj, dep_symbol.symbol_idx); + COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dep_symbol.obj, dep_symbol.symbol_idx); COFF_SymbolValueInterpType dep_interp = coff_interp_from_parsed_symbol(dep_parsed); if (dep_interp == COFF_SymbolValueInterp_Weak) { U32 member_idx; - if (lnk_search_lib(lib, symbol_parsed.name, &member_idx)) { + if (lnk_search_lib(lib, symbol->name, &member_idx)) { lnk_queue_lib_member(arena, task->imports_hm, task->link->lib_member_infos_hm, member_ref_list, symbol, lib, lib_member_infos, member_idx); } } @@ -1784,6 +1803,7 @@ lnk_link_inputs(TP_Context *tp, if (null_symbol == 0) { null_symbol = push_array(inputer->arena, LNK_Symbol, 1); null_symbol->refs = push_array(inputer->arena, LNK_ObjSymbolRefNode, 1); + null_symbol->refs_tail = null_symbol->refs; null_symbol->refs->v.obj = &link->objs.first->data; } LNK_LibMemberRef *member_refs = push_array(scratch.arena, LNK_LibMemberRef, lib->member_count); @@ -1793,16 +1813,32 @@ lnk_link_inputs(TP_Context *tp, } else { // search symbols in lib MemoryZeroTyped(member_ref_lists, tp->worker_count); - LNK_SearchLibTask search_task = { - .search_anti_deps = search_anti_deps, - .link = link, - .imports_hm = &imports_hm, - .lib = lib, - .symtab = symtab, - .lib_member_infos = lib_member_infos, - .member_ref_lists = member_ref_lists - }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); + + // barrier elision: the search task scans every undefined/weak symbol in search_chunks and + // queues any member that resolves one. search_chunks only grows during this loop and the + // dedup against already-queued members is idempotent, so if neither the symbol set nor the + // anti-dep mode changed since this lib was last searched, the dispatch can only re-queue + // (deduped to) nothing. skipping it avoids waking+joining every worker for no work. + U64 search_symbol_count = lnk_symbol_table_search_symbol_count(symtab); + B32 can_skip_search = lib->was_searched && + lib->searched_symbol_count == search_symbol_count && + lib->searched_anti_deps == search_anti_deps; + if ( ! can_skip_search) { + LNK_SearchLibTask search_task = { + .search_anti_deps = search_anti_deps, + .link = link, + .imports_hm = &imports_hm, + .lib = lib, + .symtab = symtab, + .lib_member_infos = lib_member_infos, + .member_ref_lists = member_ref_lists + }; + tp_for_parallel(tp, arena, tp->worker_count, lnk_search_lib_task, &search_task); + + lib->was_searched = 1; + lib->searched_symbol_count = search_symbol_count; + lib->searched_anti_deps = search_anti_deps; + } } LNK_LibMemberRefList queued_members = {0}; @@ -1879,7 +1915,8 @@ lnk_link_inputs(TP_Context *tp, AssertAlways(member_ref->link_symbol->refs != import_stub->refs); // replace the import symbol with a stub, which is later replaced with the real import symbol once import obj is ready. - member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs_tail = import_stub->refs_tail; // push import member for import obj generation lnk_lib_member_ref_list_push_node(&link->imports, member_ref); @@ -2001,7 +2038,15 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer String8List delayed_dll_names = {0}; String8List static_dll_names = {0}; - for EachNode(member_ref, LNK_LibMemberRef, link->imports.first) { + // sort import members into a deterministic total order before generating import + // objs: the parallel lib search appends to link->imports in nondeterministic + // discovery-round order, which would otherwise make the per-DLL import symbol + // layout (IAT slots, jump thunks) -- and every reloc against them -- nonreproducible. + LNK_LibMemberRef **import_refs = lnk_array_from_lib_member_list(scratch.arena, link->imports); + radsort(import_refs, link->imports.count, lnk_import_ref_is_before); + + for EachIndex(import_ref_idx, link->imports.count) { + LNK_LibMemberRef *member_ref = import_refs[import_ref_idx]; LNK_Lib *lib = member_ref->lib; U64 member_idx = member_ref->member_idx; LNK_LibMemberInfo *member_infos = hash_map_search_raw_raw(&link->lib_member_infos_hm, lib); @@ -2374,6 +2419,13 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer ProfEnd(); } + // + // fold identical code COMDATs (routes followers to a leader; /OPT:REF then GCs them) + // + if (config->opt_icf == LNK_SwitchState_Yes) { + lnk_opt_icf(tp, symtab, config, link->objs); + } + // // discard COMDAT sections that are not referenced // @@ -2655,7 +2707,7 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) if (section_flags & LNK_SECTION_FLAG_DEBUG) { continue; } LNK_RelocRefs refs = {0}; - refs.obj = ref_symbol.obj; + refs.obj = ref_symbol.obj; refs.relocs = lnk_coff_reloc_info_from_section_number(ref_symbol.obj, section_number); // get a batch node @@ -2762,6 +2814,538 @@ THREAD_POOL_TASK_FUNC(lnk_walk_relocs_and_mark_ref_sections_task) ProfEnd(); } +internal U64 +lnk_icf_mix(U64 h, U64 x) +{ + h ^= x; + h *= 0x100000001b3ull; + h ^= h >> 29; + return h; +} + +// Flat open-addressing U64 -> U64 map used by /OPT:ICF. The tree-based HashMap is far too slow +// for the millions of per-round dense-id lookups; this linear-probing table keeps the hot loops +// cache-friendly. Empty slots hold key == LNK_ICF_EMPTY (a value the stored hashes never take). +#define LNK_ICF_EMPTY 0xffffffffffffffffull +typedef struct LNK_ICFMap +{ + U64 *keys; + U64 *vals; + U64 mask; +} LNK_ICFMap; + +// avalanche so structured keys (e.g. (input_idx<<32)|section_number, whose low bits repeat +// across objects) scatter across slots instead of clustering into long probe chains +internal U64 +lnk_icf_scramble(U64 x) +{ + x ^= x >> 33; + x *= 0xff51afd7ed558ccdull; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ull; + x ^= x >> 33; + return x; +} + +internal LNK_ICFMap +lnk_icf_map_make(Arena *arena, U64 capacity) +{ + U64 cap = 1; + while (cap < capacity*2) { cap <<= 1; } + LNK_ICFMap m = {0}; + m.keys = push_array_no_zero(arena, U64, cap); + m.vals = push_array_no_zero(arena, U64, cap); + m.mask = cap - 1; + for EachIndex(i, cap) { m.keys[i] = LNK_ICF_EMPTY; } + return m; +} + +// look up key; returns stored value or fallback if absent +internal U64 +lnk_icf_map_get(LNK_ICFMap *m, U64 key, U64 fallback) +{ + U64 slot = lnk_icf_scramble(key) & m->mask; + for (;;) { + U64 k = m->keys[slot]; + if (k == key) { return m->vals[slot]; } + if (k == LNK_ICF_EMPTY) { return fallback; } + slot = (slot + 1) & m->mask; + } +} + +internal void +lnk_icf_map_put(LNK_ICFMap *m, U64 key, U64 val) +{ + U64 slot = lnk_icf_scramble(key) & m->mask; + for (;;) { + U64 k = m->keys[slot]; + if (k == LNK_ICF_EMPTY || k == key) { m->keys[slot] = key; m->vals[slot] = val; return; } + slot = (slot + 1) & m->mask; + } +} + +typedef struct LNK_ICFCand +{ + LNK_Obj *obj; + U32 sn; // section number + U32 reloc_first; // index into flattened reloc-target arrays + U32 reloc_count; + U64 key0; // round-0 content key + U64 color; // current equivalence class +} LNK_ICFCand; + +typedef struct LNK_ICFHashTask +{ + Rng1U64 *ranges; + LNK_ICFCand *cands; + LNK_ICFMap *cand_map; + LNK_SymbolTable *symtab; + U8 *rt_iscand; + U64 *rt_target; +} LNK_ICFHashTask; + +// per-candidate content hash + relocation-target resolution (parallel; each candidate writes +// its own disjoint reloc slice and key0, all reads are of immutable structures) +internal +THREAD_POOL_TASK_FUNC(lnk_icf_hash_task) +{ + LNK_ICFHashTask *task = raw_task; + for EachInRange(ci, task->ranges[task_id]) { + LNK_ICFCand *c = &task->cands[ci]; + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(c->obj, c->sn); + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(c->obj, header); + String8 data = str8_substr(c->obj->data, rng_1u64(header->foff, header->foff + header->fsize)); + + blake3_hasher h; blake3_hasher_init(&h); + U32 flags_for_hash = c->obj->section_flags[c->sn - 1] & ~(COFF_SectionFlag_LnkCOMDAT | COFF_SectionFlag_LnkRemove); + blake3_hasher_update(&h, &flags_for_hash, sizeof(flags_for_hash)); + blake3_hasher_update(&h, &header->fsize, sizeof(header->fsize)); + blake3_hasher_update(&h, data.str, data.size); + + for EachIndex(ri, relocs.count) { + COFF_Reloc *reloc = &relocs.v[ri]; + blake3_hasher_update(&h, &reloc->type, sizeof(reloc->type)); + blake3_hasher_update(&h, &reloc->apply_off, sizeof(reloc->apply_off)); + + COFF_ParsedSymbol tp = lnk_parsed_symbol_from_coff_symbol_idx(c->obj, reloc->isymbol); + COFF_SymbolValueInterpType ti = coff_interp_from_parsed_symbol(tp); + LNK_ObjSymbolRef tref = { c->obj, reloc->isymbol }; + if (ti == COFF_SymbolValueInterp_Undefined || ti == COFF_SymbolValueInterp_Weak) { + LNK_ObjSymbolRef resolved = {0}; + if (lnk_resolve_symbol(task->symtab, tref, &resolved)) { + tref = resolved; + tp = lnk_parsed_symbol_from_coff_symbol_idx(tref.obj, tref.symbol_idx); + ti = coff_interp_from_parsed_symbol(tp); + } + } + + U8 iscand = 0; + U64 target = 0; + if (ti == COFF_SymbolValueInterp_Regular && tref.obj != 0) { + U64 cv = lnk_icf_map_get(task->cand_map, Compose64Bit(tref.obj->input_idx, tp.section_number), 0); + if (cv) { iscand = 1; target = cv - 1; } + else { target = lnk_icf_mix(Compose64Bit(tref.obj->input_idx, tp.section_number), tp.value); } + } else { + U64 nh = 14695981039346656037ull; + for (U64 i = 0; i < tp.name.size; i += 1) { nh = lnk_icf_mix(nh, tp.name.str[i]); } + target = lnk_icf_mix(nh, tp.value); + } + + U64 idx = (U64)c->reloc_first + ri; + task->rt_iscand[idx] = iscand; + task->rt_target[idx] = target; + if (!iscand) { blake3_hasher_update(&h, &target, sizeof(target)); } + } + + U8 out[16]; blake3_hasher_finalize(&h, out, sizeof(out)); + U64 lo = *(U64 *)&out[0], hi = *(U64 *)&out[8]; + c->key0 = lnk_icf_mix(lo, hi); + } +} + +typedef struct LNK_ICFRefineTask +{ + Rng1U64 *ranges; + LNK_ICFCand *cands; + U8 *rt_iscand; + U64 *rt_target; + U64 *newkey; + U32 *active; // when set, ranges index into active[] rather than cands[] directly +} LNK_ICFRefineTask; + +// recompute each candidate's refinement key from its current color and its targets' colors +// (parallel; reads the immutable color snapshot, writes its own newkey slot) +internal +THREAD_POOL_TASK_FUNC(lnk_icf_refine_task) +{ + LNK_ICFRefineTask *task = raw_task; + for EachInRange(ai, task->ranges[task_id]) { + U64 ci = task->active ? task->active[ai] : ai; + LNK_ICFCand *c = &task->cands[ci]; + U64 k = lnk_icf_mix(0x9e3779b97f4a7c15ull, c->color); + for EachIndex(j, c->reloc_count) { + U64 idx = (U64)c->reloc_first + j; + U64 t = task->rt_iscand[idx] ? task->cands[task->rt_target[idx]].color : task->rt_target[idx]; + k = lnk_icf_mix(k, t); + } + task->newkey[ci] = k; + } +} + +// densify the keys of just the active subset into class ids drawn from `base` upward (so they never +// collide with the finalized colors of candidates that already dropped out as singletons). Returns +// the number of distinct classes among the active set. Equal keys land in one class regardless of +// sort tie order, so color values are deterministic. +internal U64 +lnk_icf_dense_colors_active(TP_Context *tp, Arena *arena, U32 *active, U64 n, U64 *newkey, LNK_ICFCand *cands, U64 base) +{ + Temp t = temp_begin(arena); + U64 *sk = push_array_no_zero(t.arena, U64, n ? n : 1); + U32 *sv = push_array_no_zero(t.arena, U32, n ? n : 1); // index into active[] + for EachIndex(i, n) { sk[i] = newkey[active[i]]; sv[i] = (U32)i; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, sk, sv); + U64 nc = 0; + for EachIndex(k, n) { + if (k == 0 || sk[k] != sk[k - 1]) { nc += 1; } + cands[active[sv[k]]].color = base + (nc - 1); + } + temp_end(t); + return nc; +} + +// Is an object section an ICF fold candidate? Only externally-defined COMDATs are folded: the +// follower is redirected at its shared symbol-table node, so every reference (including from other +// objects) resolves to the leader and /OPT:REF then drops the follower. Returns 1 = candidate. +internal U32 +lnk_icf_section_kind(LNK_Obj *obj, U64 sect_idx) +{ + COFF_SectionFlags flags = obj->section_flags[sect_idx]; + if (~flags & COFF_SectionFlag_LnkCOMDAT) { return 0; } + if ( flags & COFF_SectionFlag_LnkRemove) { return 0; } + // fold code and read-only initialized data (const tables, vtables, string literals); folding + // identical read-only data lets functions that reference it fold too (cascade). Mutable data + // is never folded. + B32 is_code = (flags & COFF_SectionFlag_CntCode) != 0; + B32 is_rodata = (flags & COFF_SectionFlag_CntInitializedData) && !(flags & COFF_SectionFlag_MemWrite); + if (!is_code && !is_rodata) { return 0; } + U32 sn = (U32)sect_idx + 1; + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(obj, sn); + if (header->fsize == 0) { return 0; } + LNK_Symbol *sym = lnk_obj_get_comdat_symlink(obj, sn); + if (sym == 0) { return 0; } // no external defining symbol (static / internal-linkage) + LNK_ObjSymbolRef sym_ref = lnk_ref_from_symbol(sym); + if (sym_ref.obj != obj) { return 0; } // same-name follower (already removed) + COFF_ParsedSymbol sym_parsed = lnk_parsed_from_symbol(sym); + if (sym_parsed.section_number != sn || sym_parsed.value != 0) { return 0; } + return 1; // external leader +} + +typedef struct LNK_ICFCollectTask +{ + LNK_Obj **objs; + U64 *counts; + U64 *offsets; + LNK_ICFCand *cands; +} LNK_ICFCollectTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_icf_count_task) +{ + LNK_ICFCollectTask *t = raw_task; + LNK_Obj *obj = t->objs[task_id]; + U64 n = 0; + for EachIndex(si, obj->header.section_count_no_null) { if (lnk_icf_section_kind(obj, si)) { n += 1; } } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_icf_fill_task) +{ + LNK_ICFCollectTask *t = raw_task; + LNK_Obj *obj = t->objs[task_id]; + U64 cur = t->offsets[task_id]; + for EachIndex(si, obj->header.section_count_no_null) { + if (lnk_icf_section_kind(obj, si)) { + LNK_ICFCand *c = &t->cands[cur++]; + c->obj = obj; c->sn = (U32)si + 1; + // count relocs here (parallel, the section is already in hand) so lnk_opt_icf only needs a + // cheap serial prefix sum for reloc_first instead of re-parsing every section serially. + COFF_SectionHeader *header = lnk_coff_section_header_from_section_number(obj, c->sn); + c->reloc_first = 0; + c->reloc_count = (U32)lnk_coff_relocs_from_section_header(obj, header).count; + c->key0 = 0; c->color = 0; + } + } +} + +// --- parallel LSD radix sort (U64 key + U32 value), 8 x 8-bit passes ------------------------- +// 8-bit digits keep the serial per-pass offset prefix tiny (256*workers, not 65536*workers); +// the extra passes are parallel histogram+scatter, so they cost little. +#define LNK_RADIX_BITS 8 +#define LNK_RADIX_SIZE (1 << LNK_RADIX_BITS) +typedef struct LNK_RadixSortTask +{ + Rng1U64 *ranges; + U64 *ksrc; U32 *vsrc; // read + U64 *kdst; U32 *vdst; // write (scatter) + U32 *hist; // [worker_count * LNK_RADIX_SIZE], per-worker digit offsets + U64 shift; +} LNK_RadixSortTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_radix_hist_task) +{ + LNK_RadixSortTask *t = raw_task; + U32 *h = t->hist + (U64)task_id * LNK_RADIX_SIZE; + for EachInRange(i, t->ranges[task_id]) { h[(t->ksrc[i] >> t->shift) & (LNK_RADIX_SIZE - 1)] += 1; } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_radix_scatter_task) +{ + LNK_RadixSortTask *t = raw_task; + U32 *h = t->hist + (U64)task_id * LNK_RADIX_SIZE; + for EachInRange(i, t->ranges[task_id]) { + U64 d = (t->ksrc[i] >> t->shift) & (LNK_RADIX_SIZE - 1); + U32 pos = h[d]++; + t->kdst[pos] = t->ksrc[i]; + t->vdst[pos] = t->vsrc[i]; + } +} + +// sort (keys[], vals[]) ascending by key, in place. arena supplies scratch (double buffers + histograms). +internal void +lnk_radix_sort_u64_pairs(TP_Context *tp, Arena *arena, U64 n, U64 *keys, U32 *vals) +{ + if (n < 2) { return; } + U64 W = tp->worker_count; + + Temp scratch = scratch_begin(&arena, 1); // internal buffers freed on return + LNK_RadixSortTask t = {0}; + t.ranges = tp_divide_work(scratch.arena, n, W); + t.hist = push_array_no_zero(scratch.arena, U32, W * LNK_RADIX_SIZE); + U64 *kbuf = push_array_no_zero(scratch.arena, U64, n); + U32 *vbuf = push_array_no_zero(scratch.arena, U32, n); + + U64 *ksrc = keys, *kdst = kbuf; + U32 *vsrc = vals, *vdst = vbuf; + for (U64 pass = 0; pass < 64 / LNK_RADIX_BITS; pass += 1) { + t.shift = pass * LNK_RADIX_BITS; + t.ksrc = ksrc; t.vsrc = vsrc; t.kdst = kdst; t.vdst = vdst; + + MemoryZero(t.hist, sizeof(U32) * W * LNK_RADIX_SIZE); + tp_for_parallel(tp, 0, W, lnk_radix_hist_task, &t); + + // exclusive prefix across (bucket, worker) so each worker writes a disjoint contiguous run + U64 running = 0; + for EachIndex(bucket, LNK_RADIX_SIZE) { + for EachIndex(w, W) { + U32 *slot = &t.hist[w * LNK_RADIX_SIZE + bucket]; + U32 c = *slot; + *slot = (U32)running; + running += c; + } + } + + tp_for_parallel(tp, 0, W, lnk_radix_scatter_task, &t); + + U64 *kt = ksrc; ksrc = kdst; kdst = kt; + U32 *vt = vsrc; vsrc = vdst; vdst = vt; + } + // even pass count -> sorted data is back in the original keys/vals arrays + scratch_end(scratch); +} + +// assign each candidate a dense equivalence-class id from its key, via a parallel sort + a +// cheap sequential group scan. Returns the number of distinct classes (for convergence). +internal U64 +lnk_icf_dense_colors(TP_Context *tp, Arena *arena, U64 n, U64 *keys, LNK_ICFCand *cands) +{ + Temp t = temp_begin(arena); + U64 *sk = push_array_no_zero(t.arena, U64, n ? n : 1); + U32 *sv = push_array_no_zero(t.arena, U32, n ? n : 1); + for EachIndex(ci, n) { sk[ci] = keys[ci]; sv[ci] = (U32)ci; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, sk, sv); + U64 nc = 0; + for EachIndex(k, n) { + if (k == 0 || sk[k] != sk[k - 1]) { nc += 1; } + cands[sv[k]].color = nc - 1; + } + temp_end(t); + return nc; +} + +// relocations point at equivalent targets (iteratively), then folds each group's followers +// into a single leader by routing them through the existing COMDAT symlink machinery and +// letting /OPT:REF garbage-collect the now-unreferenced follower sections (and their +// associated .pdata/.xdata/.debug$S). Mirrors link.exe /OPT:ICF. +internal void +lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs_list) +{ + if (config->opt_icf != LNK_SwitchState_Yes) { return; } + + ProfBeginFunction(); + Temp scratch = scratch_begin(0, 0); + Arena *arena = scratch.arena; + + U64 objs_count = objs_list.count; + LNK_Obj **objs = lnk_array_from_obj_list(arena, objs_list); + + // collect candidate sections (parallel per obj): count, exact-size, then fill at per-obj + // offsets. Counting first avoids sizing the array to the total section count (which would be + // ~10x larger and waste ~1GB on UE-scale inputs). + LNK_ICFCollectTask col = {0}; + col.objs = objs; + col.counts = push_array(arena, U64, objs_count ? objs_count : 1); + tp_for_parallel(tp, 0, objs_count, lnk_icf_count_task, &col); + col.offsets = offsets_from_counts_array_u64(arena, col.counts, objs_count); + U64 cand_count = sum_array_u64(objs_count, col.counts); + LNK_ICFCand *cands = push_array_no_zero(arena, LNK_ICFCand, cand_count ? cand_count : 1); + col.cands = cands; + tp_for_parallel(tp, 0, objs_count, lnk_icf_fill_task, &col); + + if (cand_count < 2) { goto done; } + + // build target lookup (input_idx, section_number) -> cand_idx+1, sized to the candidate count + LNK_ICFMap cand_map = lnk_icf_map_make(arena, cand_count); + for EachIndex(ci, cand_count) { + lnk_icf_map_put(&cand_map, Compose64Bit(cands[ci].obj->input_idx, cands[ci].sn), ci + 1); + } + + // assign each candidate a disjoint slice in the flattened reloc-target arrays. reloc_count was + // filled in parallel by lnk_icf_fill_task, so this is just a serial prefix sum (no re-parsing). + U64 total_relocs = 0; + for EachIndex(ci, cand_count) { + cands[ci].reloc_first = (U32)total_relocs; + total_relocs += cands[ci].reloc_count; + } + U8 *rt_iscand = push_array_no_zero(arena, U8, total_relocs ? total_relocs : 1); + U64 *rt_target = push_array_no_zero(arena, U64, total_relocs ? total_relocs : 1); + + { + LNK_ICFHashTask hash_task = {0}; + hash_task.ranges = tp_divide_work(arena, cand_count, tp->worker_count); + hash_task.cands = cands; + hash_task.cand_map = &cand_map; + hash_task.symtab = symtab; + hash_task.rt_iscand = rt_iscand; + hash_task.rt_target = rt_target; + tp_for_parallel(tp, 0, tp->worker_count, lnk_icf_hash_task, &hash_task); + } + + // assign initial colors from content key (parallel sort + group scan) + U64 *newkey = push_array_no_zero(arena, U64, cand_count ? cand_count : 1); + for EachIndex(ci, cand_count) { newkey[ci] = cands[ci].key0; } + U64 class_count = lnk_icf_dense_colors(tp, arena, cand_count, newkey, cands); + U64 color_base = class_count; // next free class id; ids only ever grow, so finalized colors stick + + // Active set = candidates still sharing a class with another. A singleton class can never split + // or merge, so once a candidate is alone its color is final and it leaves refinement. Each round + // re-densifies only the active set, shrinking the per-round sort from all ~N candidates down to + // just those that still have a content+reloc twin. + U32 *active = push_array_no_zero(arena, U32, cand_count ? cand_count : 1); + U64 active_count = 0, active_class_count = 0; + { + Temp t = temp_begin(arena); + U32 *cls_size = push_array(t.arena, U32, class_count ? class_count : 1); + for EachIndex(ci, cand_count) { cls_size[cands[ci].color] += 1; } + for EachIndex(ci, cand_count) { if (cls_size[cands[ci].color] > 1) { active[active_count++] = (U32)ci; } } + for EachIndex(c, class_count) { if (cls_size[c] > 1) { active_class_count += 1; } } + temp_end(t); + } + + // iteratively refine classes by relocation-target classes until no class splits + LNK_ICFRefineTask refine_task = {0}; + refine_task.cands = cands; + refine_task.rt_iscand = rt_iscand; + refine_task.rt_target = rt_target; + refine_task.newkey = newkey; + refine_task.active = active; + for (U64 round = 0; round < 30 && active_count > 0; round += 1) { + refine_task.ranges = tp_divide_work(arena, active_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_icf_refine_task, &refine_task); + + U64 base = color_base; + U64 nc = lnk_icf_dense_colors_active(tp, arena, active, active_count, newkey, cands, base); + color_base += nc; + if (nc == active_class_count) { break; } // no class split -> fixpoint + + // drop classes that just became singletons; keep the rest active for the next round + Temp t = temp_begin(arena); + U32 *cls_size = push_array(t.arena, U32, nc ? nc : 1); + for EachIndex(i, active_count) { cls_size[cands[active[i]].color - base] += 1; } + U64 na = 0, nac = 0; + for EachIndex(i, active_count) { if (cls_size[cands[active[i]].color - base] > 1) { active[na++] = active[i]; } } + for EachIndex(c, nc) { if (cls_size[c] > 1) { nac += 1; } } + active_count = na; active_class_count = nac; + temp_end(t); + } + + // group candidates by final color (parallel sort) and fold followers into a leader + U64 *keys = newkey; // reuse + U32 *sci = push_array_no_zero(arena, U32, cand_count ? cand_count : 1); + for EachIndex(ci, cand_count) { keys[ci] = cands[ci].color; sci[ci] = (U32)ci; } + lnk_radix_sort_u64_pairs(tp, arena, cand_count, keys, sci); + + U64 fold_count = 0; + for (U64 i = 0; i < cand_count; ) { + U64 j = i + 1; + U64 color = keys[i]; + while (j < cand_count && keys[j] == color) { j += 1; } + + if (j - i >= 2) { + // leader = lowest (input_idx, sn) member, for deterministic output + U64 leader_oi = i; + for (U64 k = i + 1; k < j; k += 1) { + LNK_ICFCand *a = &cands[sci[leader_oi]]; + LNK_ICFCand *b = &cands[sci[k]]; + if (Compose64Bit(b->obj->input_idx, b->sn) < Compose64Bit(a->obj->input_idx, a->sn)) { leader_oi = k; } + } + LNK_ICFCand *L = &cands[sci[leader_oi]]; + COFF_SectionHeader *Lheader = lnk_coff_section_header_from_section_number(L->obj, L->sn); + String8 Ldata = str8_substr(L->obj->data, rng_1u64(Lheader->foff, Lheader->foff + Lheader->fsize)); + LNK_SymbolHashTrie *Lnode = L->obj->symlinks[L->sn]; + + for (U64 k = i; k < j; k += 1) { + if (k == leader_oi) { continue; } + LNK_ICFCand *F = &cands[sci[k]]; + COFF_SectionHeader *Fheader = lnk_coff_section_header_from_section_number(F->obj, F->sn); + + // verify byte-identical content + reloc structure (guards against hash collisions) + if (Fheader->fsize != Lheader->fsize) { continue; } + if (F->reloc_count != L->reloc_count) { continue; } + String8 Fdata = str8_substr(F->obj->data, rng_1u64(Fheader->foff, Fheader->foff + Fheader->fsize)); + if (!str8_match(Ldata, Fdata, 0)) { continue; } + B32 relocs_match = 1; + for EachIndex(t, L->reloc_count) { + U64 li = L->reloc_first + t, fi = F->reloc_first + t; + U64 lt = rt_iscand[li] ? cands[rt_target[li]].color : rt_target[li]; + U64 ft = rt_iscand[fi] ? cands[rt_target[fi]].color : rt_target[fi]; + if (lt != ft) { relocs_match = 0; break; } + } + if (!relocs_match) { continue; } + + // redirect at the shared symbol-table node so every object resolving this name points at + // the leader's definition; /OPT:REF then removes the now-unreferenced follower section. + LNK_SymbolHashTrie *Fnode = F->obj->symlinks[F->sn]; + if (Fnode && Fnode != Lnode) { + Fnode->symbol = Lnode->symbol; + fold_count += 1; + } + } + } + i = j; + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "/OPT:ICF folded %llu of %llu code COMDATs into %llu classes", fold_count, cand_count, class_count); + } + + done:; + scratch_end(scratch); + ProfEnd(); +} + internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs) { @@ -2950,15 +3534,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_comdat_leaders_task) task->u.patch_symtabs.was_symbol_patched[obj_idx][symbol_idx] = 1; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } } @@ -2979,11 +3556,33 @@ lnk_section_contrib_ptr_is_before(void *raw_a, void *raw_b) return u64_compar_is_before(&input_idx_a, &input_idx_b); } +// chunks at/above this size are sorted by the parallel radix (all threads) before the per-chunk +// task pass, so one giant section (e.g. merged .text) can't serialize the whole sort on one thread. +#define LNK_SORT_CONTRIBS_RADIX_MIN (64u*1024u) + +// sort one chunk's contribs by Compose64Bit(obj_idx, obj_sect_idx) using the parallel radix sort. +// The key is unique per contrib (one obj/section pair each), so this matches the radsort order. +internal void +lnk_sort_contribs_chunk_radix(TP_Context *tp, Arena *arena, LNK_SectionContribChunk *chunk) +{ + U64 n = chunk->count; + Temp t = temp_begin(arena); + U64 *keys = push_array_no_zero(t.arena, U64, n); + U32 *idx = push_array_no_zero(t.arena, U32, n); + for EachIndex(i, n) { keys[i] = Compose64Bit(chunk->v[i]->u.obj_idx, chunk->v[i]->u.obj_sect_idx); idx[i] = (U32)i; } + lnk_radix_sort_u64_pairs(tp, t.arena, n, keys, idx); + LNK_SectionContrib **sorted = push_array_no_zero(t.arena, LNK_SectionContrib *, n); + for EachIndex(i, n) { sorted[i] = chunk->v[idx[i]]; } + MemoryCopy(chunk->v, sorted, n * sizeof(sorted[0])); + temp_end(t); +} + internal THREAD_POOL_TASK_FUNC(lnk_sort_contribs_task) { LNK_BuildImageTask *task = raw_task; LNK_SectionContribChunk *chunk = task->u.sort_contribs.chunks[task_id]; + if (chunk->count >= LNK_SORT_CONTRIBS_RADIX_MIN) { return; } // big chunks done via parallel radix ProfBeginV("[%llu]", chunk->count); radsort(chunk->v, chunk->count, lnk_section_contrib_ptr_is_before); ProfEnd(); @@ -3022,15 +3621,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_leaders_task) COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol); U64 section_number = task->u.patch_symtabs.common_block_sect->sect_idx + 1; - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = parsed_symbol.raw_symbol; - symbol32->value = contrib->u.offset; - symbol32->section_number = safe_cast_u32(section_number); - } else { - COFF_Symbol16 *symbol16 = parsed_symbol.raw_symbol; - symbol16->value = contrib->u.offset; - symbol16->section_number = safe_cast_u16(section_number); - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = contrib->u.offset; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = safe_cast_u32(section_number); task->u.patch_symtabs.was_symbol_patched[symbol_ref.obj->input_idx][symbol_ref.symbol_idx] = 1; } @@ -3055,17 +3647,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_symbols_task) COFF_ParsedSymbol defn_parsed = lnk_parsed_from_symbol(defn); Assert(lnk_interp_from_symbol(defn) == COFF_SymbolValueInterp_Regular); if (defn) { - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = defn_parsed.section_number; - symbol32->value = safe_cast_u32(defn_parsed.value); - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(defn_parsed.section_number); - symbol16->value = safe_cast_u32(defn_parsed.value); - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = defn_parsed.section_number; + obj->parsed_symbols[symbol_idx].value = defn_parsed.value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } } } @@ -3103,15 +3687,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_regular_symbols_task) value = sc->u.off + symbol.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(section_number); - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } ProfEnd(); @@ -3148,17 +3725,9 @@ lnk_patch_obj_symtab(LNK_SymbolTable *symtab, LNK_Obj *obj, B8 *was_symbol_patch value = fixup_src.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = fixup_dst.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = fixup_dst.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; was_symbol_patched[symbol_idx] = 1; } @@ -3839,17 +4408,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) if (sect && (~sect->flags & COFF_SectionFlag_LnkRemove)) { if (~sect->flags & COFF_SectionFlag_MemDiscardable) { LNK_SectionContrib *first_sc = lnk_get_first_section_contrib(sect); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = safe_cast_u32(first_sc->u.sect_idx + 1); - symbol32->value = first_sc->u.off; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(first_sc->u.sect_idx + 1); - symbol16->value = first_sc->u.off; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = safe_cast_u32(first_sc->u.sect_idx + 1); + obj->parsed_symbols[symbol_idx].value = first_sc->u.off; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } else { lnk_error_obj(LNK_Error_SectRefsDiscardedMemory, obj, "symbol %S (No. 0x%llx) references section with discard flag", symbol.name, symbol_idx); } @@ -3867,17 +4428,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) LNK_Section *fallback_sect = task->image_sects.v[task->image_sects.count-1]; U32 fallback_section_number = safe_cast_u32(fallback_sect->sect_idx + 1); U32 fallback_section_offset = safe_cast_u32(fallback_voff - fallback_sect->voff); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = fallback_section_number; - symbol32->value = fallback_section_offset; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(fallback_section_number); - symbol16->value = fallback_section_offset; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = fallback_section_number; + obj->parsed_symbols[symbol_idx].value = fallback_section_offset; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; lnk_error_obj(LNK_Warning_UndefinedSectionSymbol, obj, "undefined section symbol %S (No. 0x%llx) refers to an image section that doesn't exist; patching to %#llx", symbol.name, symbol_idx, fallback_voff); } @@ -4532,6 +5085,14 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT Assert(cursor == total_chunk_count); } + // big chunks (e.g. merged .text) first, each sorted across all threads via parallel radix -- + // otherwise a single huge chunk serializes on one worker. Small chunks then go one-per-task. + for EachIndex(ci, total_chunk_count) { + LNK_SectionContribChunk *chunk = task.u.sort_contribs.chunks[ci]; + if (chunk->count >= LNK_SORT_CONTRIBS_RADIX_MIN) { + lnk_sort_contribs_chunk_radix(tp, scratch.arena, chunk); + } + } tp_for_parallel(tp, 0, total_chunk_count, lnk_sort_contribs_task, &task); ProfEnd(); @@ -4743,7 +5304,11 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); @@ -5262,6 +5827,7 @@ lnk_write_thread(void *raw_ctx) ProfEnd(); } + internal void lnk_log_timers(void) { @@ -5350,6 +5916,52 @@ lnk_debug_filter_objs(Arena *arena, LNK_Obj **objs, U64 objs_count, U64 *count_o return debug_info_objs; } +// Parallel release of memory-mapped input file views. +// Inputs are mapped copy-on-write (PAGE_WRITECOPY/FILE_MAP_COPY); pages touched +// during linking become private-dirty and are reclaimed by the kernel in +// single-threaded process rundown at exit (~3s for a large link). Unmapping them +// in parallel before exit moves that reclaim off the serial post-exit path. +typedef struct LNK_UnmapViewTask +{ + String8 *views; +} LNK_UnmapViewTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_unmap_view_task) +{ + LNK_UnmapViewTask *task = raw_task; + String8 view = task->views[task_id]; +#if OS_WINDOWS + UnmapViewOfFile(view.str); +#elif OS_LINUX + munmap(view.str, view.size); +#endif +} + +internal void +lnk_release_input_views(TP_Context *tp, LNK_Inputer *inputer) +{ + Temp scratch = scratch_begin(0, 0); + + // collect distinct whole-file mapped views (is_thin); skip lib-member + // substrings and linkgen arena data + U64 cap = inputer->objs.count + inputer->libs.count; + String8 *views = push_array_no_zero(scratch.arena, String8, cap); + U64 count = 0; + for EachNode(n, LNK_Input, inputer->objs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + for EachNode(n, LNK_Input, inputer->libs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + + if (count > 0) { + U64 begin_us = now_time_us(); + LNK_UnmapViewTask task = { .views = views }; + tp_for_parallel(tp, 0, count, lnk_unmap_view_task, &task); + U64 end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "Released %llu input views in %.2f ms", count, (F64)(end_us - begin_us) / 1000.0); + } + + scratch_end(scratch); +} + internal void lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) { @@ -5368,7 +5980,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_SymbolTable *symtab = lnk_symbol_table_init(arena); // - // Link Image + // Link Image (group digests, if any, are synthesized + consumed inside lnk_link_image) // LNK_LinkResult link = lnk_link_image(tp, arena, config, inputer, symtab); @@ -5427,6 +6039,13 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: + // it removes types that a debugger can still legitimately cast to in the watch window + // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. + if (config->opt_gc_types == LNK_SwitchState_Yes) { + lnk_gc_types(tp, arena->v[0], &cv, &cv_types); + } + // // Debug Info // @@ -5545,6 +6164,20 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); + + // outputs are written and inputs are no longer read; release the copy-on-write + // input views in parallel so their dirty pages are reclaimed here (multi-threaded) + // instead of in single-threaded process rundown at exit. Only safe for the CoW + // (read-only) mapping mode; read-write-shared would flush dirty pages back to the + // input files on unmap. + if ((config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + lnk_release_input_views(tp, inputer); + } + // // Timers // @@ -5839,8 +6472,8 @@ entry_point(CmdLine *cmdline) } switch (config->boot_mode) { - case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; - case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; + case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; + case LNK_BootMode_TypeServer: lnk_run_type_server (tp, tp_arena, config); break; } lnk_log_end(); diff --git a/src/linker/lnk.h b/src/linker/lnk.h index d6388d0a0..b442a9b61 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -391,6 +391,7 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs); +internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_ObjList objs); // --- Win32 Image ------------------------------------------------------------- diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index 3d27012c3..58a24149c 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -119,6 +119,7 @@ global read_only struct { "obj", LNK_Input_Obj }, { "lib", LNK_Input_Lib }, { "rlib", LNK_Input_Lib }, // rust libs + { "a", LNK_Input_Lib }, // GNU ar archives (clang/meson, e.g. libdav1d.a) { "res", LNK_Input_Res }, { "rrt", LNK_Input_RRT }, }; @@ -1678,6 +1679,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List config->opt_lbr = LNK_SwitchState_Yes; } else if (str8_match_lit("nolibr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_No; + } else if (str8_match_lit("gctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_Yes; + } else if (str8_match_lit("nogctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_No; } else { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown option \"%S\"", param); } diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 0b6d79641..548b5cc1a 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -290,6 +290,7 @@ typedef struct LNK_Config LNK_SwitchState opt_ref; LNK_SwitchState opt_icf; LNK_SwitchState opt_lbr; + LNK_SwitchState opt_gc_types; // /OPT:GCTYPES -- prune unreferenced CodeView types. Default OFF: shrinks PDB but a pruned type can't be cast-to in the debugger watch window. U64 opt_iter_count; LNK_SwitchState import_table_emit_biat; LNK_SwitchState import_table_emit_uiat; diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 83f574969..c48e323bc 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1374,29 +1374,23 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } -internal LNK_LeafRef * -lnk_leaf_hash_table_search(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +// Returns the type index assigned to leaf_ref's deduped class, or 0 if absent. Dedup is by hash +// (lnk_match_leaf_ref is a_hash==b_hash), so the assigned-ti table is keyed purely by hash -- a single +// probe, deref-free (the occupant hash is on the slot), and sized to the UNIQUE leaf count. +internal CV_TypeIndex +lnk_leaf_hash_table_search_ti(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - LNK_LeafRef *match = 0; - - CV_DebugT *debug_t = &input->debug_t_arr[leaf_ref.obj_idx]; - CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; - U64 hash = debug_h->v[leaf_ref.leaf_idx]; - U64 best_bucket_idx = hash % ht->cap; - U64 bucket_idx = best_bucket_idx; + CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; + U64 hash = debug_h->v[leaf_ref.leaf_idx]; + U64 best_idx = hash % ht->cap; + U64 idx = best_idx; do { - LNK_LeafRef *bucket = ht->bucket_arr[bucket_idx]; - if (bucket == 0) { break; } - - if (lnk_match_leaf_ref(input, *bucket, leaf_ref)) { - match = bucket; - break; - } - - bucket_idx = (bucket_idx + 1) == ht->cap ? 0 : (bucket_idx + 1); - } while (bucket_idx != best_bucket_idx); + if (ht->ti_arr[idx] == 0) { break; } // empty slot -> not present + if (ht->hash_arr[idx] == hash) { return ht->ti_arr[idx]; } + idx = (idx + 1) == ht->cap ? 0 : (idx + 1); + } while (idx != best_idx); - return match; + return 0; } internal @@ -1748,57 +1742,39 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) { LNK_MergeTypes *task = raw_task; - CV_TypeIndexSource ti_source = task->ti_source; - LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; - CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; - U64 assigned_type_cap = task->assigned_type_caps[ti_source]; - CV_TypeIndex *assigned_type_ht = task->assigned_type_hts[ti_source]; + CV_TypeIndexSource ti_source = task->ti_source; + LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; + CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; + LNK_AssignedTiHash *at = &task->assigned_ti_arr[ti_source]; + CV_DebugH *debug_h_arr = task->input->debug_h_arr; + // Insert each unique leaf's assigned type index into the (unique-sized) hash->ti table, keyed by leaf + // hash. Unique leaves have distinct hashes (dedup is by hash), so each claims its own empty slot. + // search_ti (in the later fixup phase, after this barrier) recovers ti in one deref-free probe. for EachInRange(i, task->ranges[task_id]) { LNK_LeafRef *leaf_ref = unique_leaf_refs.v[i]; CV_TypeIndex type_index = min_type_index + i; - U64 hash = u64_hash_from_str8(str8_struct(leaf_ref)); - U64 best_idx = hash % assigned_type_cap; + U64 hash = debug_h_arr[leaf_ref->obj_idx].v[leaf_ref->leaf_idx]; + U64 best_idx = hash % at->cap; U64 idx = best_idx; - B32 is_inserted = 0; + B32 is_assigned = 0; do { - CV_TypeIndex curr_type_index = assigned_type_ht[idx]; - if (curr_type_index == 0) { - CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(&assigned_type_ht[idx], type_index, curr_type_index); - if (cmp_type_index == curr_type_index) { - is_inserted = 1; + if (at->ti_arr[idx] == 0) { + CV_TypeIndex cmp = ins_atomic_u32_eval_cond_assign(&at->ti_arr[idx], type_index, 0); + if (cmp == 0) { + at->hash_arr[idx] = hash; // only this worker owns the slot now; read back in the fixup phase + is_assigned = 1; break; } } - // advance - idx = (idx + 1) == assigned_type_cap ? 0 : (idx + 1); + idx = (idx + 1) == at->cap ? 0 : (idx + 1); } while (idx != best_idx); - Assert(is_inserted); + Assert(is_assigned); } } -internal CV_TypeIndex -lnk_assigned_type_ht_search(U64 cap, CV_TypeIndex *ht, CV_TypeIndex min_type_index, LNK_LeafRefArray unique_leaf_refs, LNK_LeafRef *v, U64 hash) -{ - U64 best_idx = hash % cap; - U64 idx = best_idx; - do { - CV_TypeIndex type_index = ht[idx]; - if (type_index < min_type_index) { break; } - - U64 leaf_idx = type_index - min_type_index; - LNK_LeafRef *compar = unique_leaf_refs.v[leaf_idx]; - if (MemoryMatchStruct(compar,v)) { return type_index; } - - idx = (idx + 1) == cap ? 0 : (idx + 1); - } while(idx != best_idx); - - InvalidPath; - return 0; -} - internal void lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) { @@ -1809,21 +1785,11 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ // skip basic types if (ti < ctx->input->min_type_indices[n->source]) { continue; } - CV_TypeIndex final_ti = 0; - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, ctx->input, leaf_ref); - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - final_ti = lnk_assigned_type_ht_search(ctx->assigned_type_caps [n->source], - ctx->assigned_type_hts [n->source], - ctx->min_type_indices [n->source], - ctx->unique_leaf_refs_arr[n->source], - final_leaf, - final_hash); - } + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); + LNK_AssignedTiHash *assigned = &ctx->assigned_ti_arr[n->source]; + CV_TypeIndex final_ti = lnk_leaf_hash_table_search_ti(assigned, ctx->input, leaf_ref); #if BUILD_DEBUG - else { + if (final_ti == 0) { lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); } #endif @@ -1992,23 +1958,9 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) for EachIndex(leaf_idx, debug_t->count) { CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); - LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; - LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, input, leaf_ref); - - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - CV_TypeIndex final_ti = lnk_assigned_type_ht_search(task->assigned_type_caps [source], - task->assigned_type_hts [source], - task->min_type_indices [source], - task->unique_leaf_refs_arr[source], - final_leaf, - final_hash); - - obj_ti_map[leaf_idx] = final_ti; - } else { - obj_ti_map[leaf_idx] = 0; - } + LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; + LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; + obj_ti_map[leaf_idx] = lnk_leaf_hash_table_search_ti(assigned, input, leaf_ref); } task->result.obj_ti_maps[obj_idx] = obj_ti_map; @@ -2152,6 +2104,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.unique_leaf_refs_arr[ti_source].count = sum_array_u64(tp->worker_count, task.counts[ti_source]); task.unique_leaf_refs_arr[ti_source].v = push_array_no_zero(scratch.arena, LNK_LeafRef *, task.unique_leaf_refs_arr[ti_source].count); + + // assigned-ti table sized to the unique (deduped) count -- not the total leaf count, which would + // add the bucket-parallel ti/hash arrays' worth of peak working set (~3GB on large links) + task.assigned_ti_arr[ti_source].cap = 1 + ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); // * 1.3 + task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); + task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); @@ -2250,8 +2208,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ProfBegin("Assign type indices"); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; - task.assigned_type_caps[ti_source] = (task.unique_leaf_refs_arr[ti_source].count * 13) / 10; - task.assigned_type_hts [ti_source] = push_array(scratch.arena, CV_TypeIndex, task.assigned_type_caps[ti_source]); task.min_type_indices [ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_assign_type_indices_task, &task, "Assign Type Indices"); @@ -3135,6 +3091,321 @@ THREAD_POOL_TASK_FUNC(lnk_push_dbi_sec_contrib_task) } } +//////////////////////////////// +// Type Garbage Collection +// +// After type merging, prune merged TPI/IPI leaves that are not reachable from any surviving +// symbol record (the GC roots), compact them, and remap all type indices. link.exe keeps a +// large unreferenced-type set; pruning it is a transparent PDB-size win (debug-info only -- the +// image is untouched). Runs on the final post-fixup type indices in place. + +typedef struct LNK_GCTypes +{ + LNK_CodeViewInput *cv; + U64 min [CV_TypeIndexSource_COUNT]; // first type index per source + U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source + U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) + CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index + U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays + U32 *udt_next; // TPI fwdref<->definition unique_name ring + Rng1U64 *sym_ranges; + B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // transitive-closure frontier: indices marked but not yet expanded. Each leaf is appended once + // (the atomic mark gates it), so frontier[s] is sized orig_n[s] and fcount[s] is its atomic tail. + U32 *frontier[CV_TypeIndexSource_COUNT]; + U32 *fcount [CV_TypeIndexSource_COUNT]; // atomic append cursor per source + // per-source scratch (set before dispatch) + CV_TypeIndexSource cur_source; + U8 **cur_leaf_v; + Rng1U64 *cur_ranges; + U64 round_begin, round_end; // frontier slice processed this round +} LNK_GCTypes; + +typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; + +internal int +lnk_gc_name_pair_is_before(void *raw_a, void *raw_b) +{ + LNK_GCNamePair *a = raw_a, *b = raw_b; + return a->hash != b->hash ? (a->hash < b->hash) : (a->idx < b->idx); +} + +internal void +lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) +{ + U64 lo = g->min[s]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[s]) { g->mark[s][idx] = 1; } } +} + +// walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) +internal void +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TypeIndexInfoList ti_info_list) +{ + for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { + U8 *p = data.str + n->offset; + CV_TypeIndex ti = memory_read32(p); + if (g->do_rewrite) { + U64 lo = g->min[n->source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n->source]) { memory_write32(p, g->remap[n->source][idx]); } } + } else { + lnk_gc_mark_ti(g, n->source, ti); + } + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->sym_ranges[task_id]) { + LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + Temp temp = temp_begin(scratch.arena); + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + CV_TypeIndexInfoList l = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); + lnk_gc_visit_offsets(g, symbol.data, l); + temp_end(temp); + } + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) +{ + LNK_GCTypes *g = raw_task; + U64 obj_idx = task_id; + Temp scratch = scratch_begin(0, 0); + String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + for EachNode(dn, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(scratch.arena); + CV_TypeIndexInfoList l = cv_get_inlinee_type_index_offsets(temp.arena, dn->string); + lnk_gc_visit_offsets(g, dn->string, l); + temp_end(temp); + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + lnk_gc_visit_offsets(g, leaf.data, l); + temp_end(temp); + } + scratch_end(scratch); +} + +typedef struct LNK_GCRingTask +{ + U8 **leaf_v; + Rng1U64 *ranges; + U64 *counts; + U64 *offsets; + LNK_GCNamePair *pairs; +} LNK_GCRingTask; + +// parallel: count UDT leaves with a unique_name per range (pass 0) / emit (hash,idx) pairs (pass 1) +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_count_task) +{ + LNK_GCRingTask *t = raw_task; + U64 n = 0; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { n += 1; } + } + } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) +{ + LNK_GCRingTask *t = raw_task; + U64 cur = t->offsets[task_id]; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { + U64 h = 14695981039346656037ull; + for EachIndex(c, ui.unique_name.size) { h = (h ^ ui.unique_name.str[c]) * 0x100000001b3ull; } + t->pairs[cur].hash = h; t->pairs[cur].idx = (U32)i; cur += 1; + } + } + } +} + +// mark a leaf reachable and, if this is its first mark, append it to its source's frontier so a +// later round expands it. Atomic mark gates the append, so each leaf lands on the frontier once. +internal void +lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) +{ + if (ci >= g->orig_n[ns]) { return; } + if (g->mark[ns][ci]) { return; } // fast non-atomic skip: already reachable (the common edge) + // only the worker that wins the 0->1 transition appends, so the atomic runs once per leaf (not + // once per reference edge). + if (!ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; + g->frontier[ns][pos] = (U32)ci; + } +} + +// one bulk-synchronous round of transitive closure: expand the frontier slice [round_begin, +// round_end) of cur_source -- visit each leaf and enqueue the leaves it references (and its +// unique_name UDT counterparts). Frontier-driven, so total work is O(reachable leaves), not +// O(rounds * total leaves). +internal +THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) +{ + LNK_GCTypes *g = raw_task; + CV_TypeIndexSource s = g->cur_source; + Temp scratch = scratch_begin(0, 0); + for EachInRange(local, g->cur_ranges[task_id]) { + U32 i = g->frontier[s][g->round_begin + local]; + + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); + CV_TypeIndexInfoList l = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); + for EachNode(n, CV_TypeIndexInfo, l.first) { + CV_TypeIndex ti = memory_read32(leaf.data.str + n->offset); + U64 lo = g->min[n->source]; + if (ti >= lo) { lnk_gc_mark_enqueue(g, n->source, ti - lo); } + } + temp_end(temp); + + if (s == CV_TypeIndexSource_TPI) { + for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { + lnk_gc_mark_enqueue(g, CV_TypeIndexSource_TPI, j); + } + } + } + scratch_end(scratch); +} + +internal void +lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + + LNK_GCTypes g = {0}; + g.cv = cv; + U64 total_leaves = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.min[s] = types->min_type_indices[s]; + g.orig_n[s] = types->count[s]; + g.leaf_v[s] = types->v[s]; + g.mark[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); // zeroed + total_leaves += g.orig_n[s]; + } + + // mark roots: every type index referenced by a surviving symbol / inlinee record + g.do_rewrite = 0; + g.sym_ranges = tp_divide_work(scratch.arena, cv->symbol_input_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + + // link UDT leaves that share a unique_name into rings, so that marking any one (e.g. a + // forward ref reached as a member-pointer target) also keeps its full definition -- needed + // for the debugger to complete types referenced only by name. TPI only (IPI has no UDTs). + U64 n_tpi = g.orig_n[CV_TypeIndexSource_TPI]; + g.udt_next = push_array_no_zero(scratch.arena, U32, n_tpi ? n_tpi : 1); + for EachIndex(i, n_tpi) { g.udt_next[i] = (U32)i; } + { + LNK_GCRingTask rt = {0}; + rt.leaf_v = g.leaf_v[CV_TypeIndexSource_TPI]; + rt.ranges = tp_divide_work(scratch.arena, n_tpi, tp->worker_count); + rt.counts = push_array(scratch.arena, U64, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_count_task, &rt); + rt.offsets = offsets_from_counts_array_u64(scratch.arena, rt.counts, tp->worker_count); + U64 np = sum_array_u64(tp->worker_count, rt.counts); + rt.pairs = push_array_no_zero(scratch.arena, LNK_GCNamePair, np ? np : 1); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_fill_task, &rt); + + radsort(rt.pairs, np, lnk_gc_name_pair_is_before); + for (U64 a = 0; a < np; ) { + U64 b = a + 1; + while (b < np && rt.pairs[b].hash == rt.pairs[a].hash) { b += 1; } + for (U64 k = a; k < b; k += 1) { g.udt_next[rt.pairs[k].idx] = rt.pairs[(k + 1 < b) ? (k + 1) : a].idx; } + a = b; + } + } + + // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each + // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new + // seed the frontier with the root-marked leaves (one O(total leaves) scan), then expand + // frontier slices until both sources drain. Each leaf is expanded exactly once. + U32 fcount[CV_TypeIndexSource_COUNT] = {0}; + U64 start [CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.frontier[s] = push_array_no_zero(scratch.arena, U32, g.orig_n[s] ? g.orig_n[s] : 1); + g.fcount[s] = &fcount[s]; + for EachIndex(i, g.orig_n[s]) { if (g.mark[s][i]) { g.frontier[s][fcount[s]++] = (U32)i; } } + } + for (;;) { + B32 any = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + U64 begin = start[s], end = fcount[s]; // fcount may grow during the round (cross-source enqueues) + if (begin < end) { + any = 1; + g.cur_source = (CV_TypeIndexSource)s; + g.round_begin = begin; + g.round_end = end; + g.cur_ranges = tp_divide_work(scratch.arena, end - begin, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + start[s] = end; + } + } + if (!any) { break; } + } + + // compact each source: assign new contiguous type indices to the kept leaves. The leaf + // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never + // passes the read cursor) and remap lives in scratch -- so the GC adds nothing to the arena + // that survives into the (peak) PDB build. + U64 kept_total = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.remap[s] = push_array_no_zero(scratch.arena, CV_TypeIndex, g.orig_n[s] ? g.orig_n[s] : 1); + U8 **v = g.leaf_v[s]; + U64 new_n = 0; + for EachIndex(idx, g.orig_n[s]) { + if (g.mark[s][idx]) { g.remap[s][idx] = (CV_TypeIndex)(g.min[s] + new_n); v[new_n++] = v[idx]; } + else { g.remap[s][idx] = 0; /* T_NOTYPE; never referenced by a kept record */ } + } + types->count[s] = new_n; + kept_total += new_n; + } + + // rewrite all type-index references to the compacted indices + g.do_rewrite = 1; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_leaf_v = types->v[s]; + g.cur_ranges = tp_divide_work(scratch.arena, types->count[s], tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_rewrite_leaves_task, &g); + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "type GC: kept %llu of %llu leaves (pruned %llu)", kept_total, total_leaves, total_leaves - kept_total); + } + + scratch_end(scratch); + ProfEnd(); +} + internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags) { diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index dc094e0f9..ee0e89e27 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -115,10 +115,20 @@ typedef struct { U64 count; LNK_LeafRef **v; } LNK_LeafRefArray; typedef struct { - U64 cap; + U64 cap; // ~1.3x total (pre-dedup) leaf count LNK_LeafRef **bucket_arr; } LNK_LeafHashTable; +// Maps leaf hash -> assigned type index, sized to the UNIQUE (post-dedup) leaf count rather than the +// total leaf count -- folds the canonical-bucket + bucket->ti lookups into one probe (deref-free: the +// hash is stored on the slot) without the total-sized per-bucket arrays that would add ~3GB to peak. +typedef struct +{ + U64 cap; // ~1.3x unique leaf count + CV_TypeIndex *ti_arr; // assigned type index per slot; 0 == empty (ti is always >= CV_MinComplexTypeIndex) + U64 *hash_arr; // occupant leaf hash (parallel to ti_arr); disambiguates open-addressing collisions +} LNK_AssignedTiHash; + typedef struct LNK_LeafRange { struct LNK_LeafRange *next; @@ -150,6 +160,7 @@ typedef struct LNK_CodeViewInput *input; CV_DebugS *debug_s_arr; LNK_LeafHashTable leaf_ht_arr[CV_TypeIndexSource_COUNT]; + LNK_AssignedTiHash assigned_ti_arr[CV_TypeIndexSource_COUNT]; Arena **fixed_arenas; CV_TypeIndexSource ti_source; U32Array indices; @@ -174,8 +185,6 @@ typedef struct U64 pass_idx; // assign type indices - U64 assigned_type_caps [CV_TypeIndexSource_COUNT]; - CV_TypeIndex *assigned_type_hts [CV_TypeIndexSource_COUNT]; CV_TypeIndex min_type_indices [CV_TypeIndexSource_COUNT]; LNK_LeafRefArray unique_leaf_refs_arr[CV_TypeIndexSource_COUNT]; @@ -251,11 +260,12 @@ internal B32 lnk_match_leaf_ref (LNK_CodeViewInput internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); -internal LNK_LeafRef * lnk_leaf_hash_table_search (LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal CV_TypeIndex lnk_leaf_hash_table_search_ti (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); // returns assigned ti for leaf_ref's hash class, 0 if absent internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); //////////////////////////////// // PDB +internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); internal String8List lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PDB_BuilderFlags builder_flags); diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index eed5e2dea..9a939c163 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -15,6 +15,13 @@ typedef struct LNK_Lib String8Array symbol_names; String8 long_names; U64 input_idx; + + // lib-search barrier elision: a re-search of this lib can only queue new members if the set of + // undefined/weak symbols grew, or if anti-dep searching was just enabled, since the last search. + // these record the state at the last search so identical re-searches can skip the tp dispatch. + B32 was_searched; + B32 searched_anti_deps; + U64 searched_symbol_count; } LNK_Lib; typedef struct LNK_LibNode diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index dabd77f83..fcc541c5e 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -125,13 +125,16 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } // - // error check symbol table + // error check symbol table (+ memoize parsed symbols) // + LNK_ParsedSymbolLite *parsed_symbols = push_array(arena, LNK_ParsedSymbolLite, header.symbol_count); { COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(input->data, header.section_table_range).str; COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = coff_parse_symbol(header, raw_coff_string_table, raw_coff_symbol_table, symbol_idx); + U32 raw_off = symbol.raw_symbol ? safe_cast_u32((U8 *)symbol.raw_symbol - input->data.str) : 0; + parsed_symbols[symbol_idx] = (LNK_ParsedSymbolLite){ raw_off, safe_cast_u32(symbol.value), symbol.section_number, symbol.type, symbol.storage_class, symbol.aux_symbol_count }; COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); if (interp == COFF_SymbolValueInterp_Regular) { if (symbol.section_number == 0 || symbol.section_number > header.section_count_no_null) { @@ -335,6 +338,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) obj->path = push_str8_copy(arena, input->path); obj->header = header; obj->section_flags = section_flags; + obj->parsed_symbols = parsed_symbols; obj->comdats = comdats; obj->exclude_from_debug_info = input->exclude_from_debug_info; obj->hotpatch = hotpatch; @@ -570,8 +574,11 @@ lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx) return sect_idx+1; } +// NOTE: skips section.name (coff_name_from_section_header does a string-table +// lookup); use when only the header/flags/ranges are needed (e.g. checking +// section flags). Callers that need the name should use lnk_obj_section_from_sect_idx. internal LNK_ObjSection -lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx) { Assert(sect_idx < obj->header.section_count_no_null); LNK_ObjSection section = {0}; @@ -580,13 +587,20 @@ lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) section.section_number = sect_idx+1; section.header = &lnk_coff_section_table_from_obj(obj)[sect_idx]; section.flags = &obj->section_flags[sect_idx]; - section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); section.vrange = rng_1u64(section.header->voff, section.header->voff + section.header->vsize); section.frange = rng_1u64(section.header->foff, section.header->foff + section.header->fsize); section.reloc_count = section.header->reloc_count; return section; } +internal LNK_ObjSection +lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +{ + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); + return section; +} + internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number) { @@ -647,19 +661,35 @@ lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number) return §ion_table[sect_idx]; } +// NOTE: returns the memoized parse built in lnk_obj_initer. The struct is mutable: symbol-value +// patching (lnk_patch_*_task) writes value/section_number/storage_class here, NOT into the mmapped +// obj->data symbol table -- so obj->data symbol values are never written after load. raw_symbol still +// points into obj->data (read-only) for aux-record reads (coff_parse_weak_tag / coff_parse_secdef). internal COFF_ParsedSymbol -lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) +lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) { - String8 string_table = str8_substr(obj->data, obj->header.string_table_range); - String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - + LNK_ParsedSymbolLite *lite = &obj->parsed_symbols[symbol_idx]; COFF_ParsedSymbol result = {0}; - if (obj->header.is_big_obj) { - result = coff_parse_symbol32(string_table, (COFF_Symbol32 *)symbol_table.str + symbol_idx); - } else { - result = coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); + result.value = lite->value; + result.section_number = lite->section_number; + result.type = lite->type; + result.storage_class = lite->storage_class; + result.aux_symbol_count = lite->aux_symbol_count; + result.raw_symbol = lite->raw_symbol_off ? (obj->data.str + lite->raw_symbol_off) : 0; // offset -> ptr + return result; +} + +internal COFF_ParsedSymbol +lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) +{ + COFF_ParsedSymbol result = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, symbol_idx); + // name is excluded from the memo -- decode it from the (read-only) symbol record on demand. Patching + // never touches the name, so re-deriving from raw_symbol stays correct after value/section patches. + if (result.raw_symbol) { + String8 string_table = str8_substr(obj->data, obj->header.string_table_range); + if (obj->header.is_big_obj) { result.name = coff_parse_symbol32(string_table, (COFF_Symbol32 *)result.raw_symbol).name; } + else { result.name = coff_parse_symbol16(string_table, (COFF_Symbol16 *)result.raw_symbol).name; } } - return result; } @@ -756,8 +786,10 @@ lnk_raw_directives_from_obj(Arena *arena, LNK_Obj *obj) { String8List drectve_data = {0}; for (U64 sect_idx = 0; sect_idx < obj->header.section_count_no_null; sect_idx += 1) { - LNK_ObjSection section = lnk_obj_section_from_sect_idx(obj, sect_idx); + // only LnkInfo sections (rare) need the name; skip the string-table lookup for the rest + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); if (*section.flags & COFF_SectionFlag_LnkInfo) { + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); if (str8_match(section.name, str8_lit(".drectve"), 0)) { if (*section.flags & COFF_SectionFlag_CntUninitializedData) { lnk_error_obj(LNK_Error_IllData, obj, ".drectve section header has flag COFF_SectionFlag_CntUninitializedData"); diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index d13a3d1e1..95df7bde7 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -5,6 +5,21 @@ // --- Input ------------------------------------------------------------------- +// Slim memoized symbol parse: every COFF_ParsedSymbol field EXCEPT name (the 16B String8). Sized by +// total symbol count, so dropping name saves ~16B/sym of peak. The name is the cold path -- it is +// re-decoded from raw_symbol on demand in lnk_parsed_symbol_from_coff_symbol_idx (the named accessor); +// the hot _no_name accessor and all symbol-value patching never touch it. +typedef struct LNK_ParsedSymbolLite +{ + U32 raw_symbol_off; // byte offset of the COFF symbol record within obj->data (0 = none). + // stored as an offset, not a pointer, to keep this struct 16B. + U32 value; // COFF symbol value is U32 (section-relative offset / size / etc.) + U32 section_number; + COFF_SymbolType type; // U16 + COFF_SymStorageClass storage_class; // U8 + U8 aux_symbol_count; +} LNK_ParsedSymbolLite; + typedef struct LNK_Obj { String8 path; @@ -12,6 +27,8 @@ typedef struct LNK_Obj COFF_FileHeaderInfo header; COFF_SectionFlags *section_flags; + LNK_ParsedSymbolLite *parsed_symbols; // memoized parse per symbol_idx (aux slots zeroed), name excluded. + // Mutable: symbol-value patching writes here, NOT obj->data. // flags B8 hotpatch; @@ -146,9 +163,11 @@ internal LNK_Symbol * lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_n internal COFF_SectionHeader * lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx); +internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx); internal U64 lnk_obj_sect_idx_from_section_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx); +internal LNK_ObjSection lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 6cd8734ef..cb02c3ce2 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -11,6 +11,7 @@ lnk_make_symbol(Arena *arena, String8 name, LNK_Obj *obj, U32 symbol_idx) LNK_Symbol *symbol = push_array(arena, LNK_Symbol, 1); symbol->name = name; symbol->refs = ref; + symbol->refs_tail = ref; return symbol; } @@ -105,12 +106,14 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) { B32 can_replace = 0; - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_ParsedSymbol src_parsed = lnk_parsed_from_symbol(src); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); - COFF_SymbolValueInterpType src_interp = lnk_interp_from_symbol(src); + // only scalar fields + raw_symbol are needed here (names come from dst->name/src->name); + // skip the symbol-name string-table scan and avoid re-parsing for interp LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); LNK_ObjSymbolRef src_ref = lnk_ref_from_symbol(src); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_ParsedSymbol src_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(src_ref.obj, src_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); + COFF_SymbolValueInterpType src_interp = coff_interp_from_parsed_symbol(src_parsed); LNK_Obj *dst_obj = dst_ref.obj; LNK_Obj *src_obj = src_ref.obj; @@ -332,9 +335,10 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) internal void lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) { - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); + // only scalar fields are needed below, so skip the symbol-name string-table scan LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); if (dst_interp == COFF_SymbolValueInterp_Regular) { // remove replaced section from the output @@ -350,10 +354,9 @@ lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) } } - // merge symbol refs - LNK_ObjSymbolRefNode *src_last_ref; - for (src_last_ref = src->refs; src_last_ref->next != 0; src_last_ref = src_last_ref->next); - src_last_ref->next = dst->refs; + // merge symbol refs (append dst's list onto src's tail; tail pointer keeps this O(1)) + src->refs_tail->next = dst->refs; + src->refs_tail = dst->refs_tail; // assert leader section is live #if BUILD_DEBUG @@ -532,7 +535,9 @@ lnk_parsed_from_symbol(LNK_Symbol *symbol) internal COFF_SymbolValueInterpType lnk_interp_from_symbol(LNK_Symbol *symbol) { - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // interp only needs scalar fields; skip the symbol-name string-table scan + LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref.obj, ref.symbol_idx); return coff_interp_from_parsed_symbol(symbol_parsed); } @@ -557,6 +562,7 @@ lnk_symbol_table_push_(LNK_SymbolTable *symtab, Arena *arena, U64 worker_id, LNK { U64 hash = lnk_symbol_table_hasher(symbol->name); COFF_SymbolValueInterpType interp = lnk_interp_from_symbol(symbol); + symbol->interp = interp; // cache for the lib search hot loop LNK_SymbolHashTrieChunkList *chunks; if (interp == COFF_SymbolValueInterp_Weak || interp == COFF_SymbolValueInterp_Undefined) { chunks = &symtab->search_chunks[worker_id]; @@ -586,6 +592,21 @@ lnk_symbol_table_search(LNK_SymbolTable *symtab, String8 name) return trie ? trie->symbol : 0; } +internal U64 +lnk_symbol_table_search_symbol_count(LNK_SymbolTable *symtab) +{ + // total number of weak/undefined symbols inserted into search_chunks. this only ever grows + // during the lib-search loop (symbols are never removed mid-loop), so it serves as a monotonic + // version stamp for the set of symbols the lib search would scan. + U64 count = 0; + for EachIndex(worker_id, symtab->arena->count) { + for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[worker_id].first) { + count += c->count; + } + } + return count; +} + internal LNK_Symbol * lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...) { @@ -773,17 +794,9 @@ THREAD_POOL_TASK_FUNC(lnk_replace_weak_with_default_symbol_task) COFF_SymbolValueInterpType resolve_interp = coff_interp_from_parsed_symbol(resolve_parsed); if (resolve_interp == COFF_SymbolValueInterp_Weak) { COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(resolve_parsed, symbol_ref.obj->header.is_big_obj); - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol_parsed.raw_symbol; - symbol32->section_number = COFF_Symbol_UndefinedSection; - symbol32->value = 0; - symbol32->storage_class = COFF_SymStorageClass_External; - } else { - COFF_Symbol16 *symbol16 = symbol_parsed.raw_symbol; - symbol16->section_number = COFF_Symbol_UndefinedSection; - symbol16->value = 0; - symbol16->storage_class = COFF_SymStorageClass_External; - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = COFF_Symbol_UndefinedSection; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = 0; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].storage_class = COFF_SymStorageClass_External; } else { symbol->refs->v = resolve; } diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index cdf383045..4fbcfcf09 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -19,8 +19,10 @@ typedef struct LNK_ObjSymbolRefNode typedef struct LNK_Symbol { - String8 name; - LNK_ObjSymbolRefNode *refs; + String8 name; + LNK_ObjSymbolRefNode *refs; + LNK_ObjSymbolRefNode *refs_tail; // tail of `refs`, so symbol-ref merges are O(1) instead of O(n) tail walks + COFF_SymbolValueInterpType interp; // cached at push so the lib search can skip re-parsing (and page-faulting) resolved symbols } LNK_Symbol; // --- Symbol Containers ------------------------------------------------------- @@ -123,6 +125,7 @@ internal U64 lnk_symbol_table_hasher(String8 string); internal LNK_SymbolTable * lnk_symbol_table_init(TP_Arena *arena); internal void lnk_symbol_table_push(LNK_SymbolTable *symtab, LNK_Symbol *symbol); internal LNK_Symbol * lnk_symbol_table_search(LNK_SymbolTable *symtab, String8 name); +internal U64 lnk_symbol_table_search_symbol_count(LNK_SymbolTable *symtab); internal LNK_Symbol * lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...); // --- Symbol Contrib Helpers -------------------------------------------------- diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 0c6e58095..ba7d5ace9 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1783,7 +1783,11 @@ psi_addr_map_compar_is_before(void *raw_a, void *raw_b) } else if (a->isect_off.off != b->isect_off.off) { is_before = a->isect_off.off < b->isect_off.off; } else { - is_before = str8_compar_case_sensitive(&a->name, &b->name) < 0; + int cmp = str8_compar_case_sensitive(&a->name, &b->name); + // unique, element-stable tiebreaker (offset is unique per record and travels with the + // element across radsort swaps) so equal-key runs (many ICF-folded symbols at one + // address) do not degrade radsort's quicksort to O(n^2) + is_before = cmp != 0 ? (cmp < 0) : (a->offset < b->offset); } return is_before; @@ -1797,12 +1801,22 @@ gsi_record_sort_by_name(PDB_GsiSortRecord *arr, U64 count) ProfEnd(); } -internal void -gsi_record_sort_by_sc(PDB_GsiSortRecord *arr, U64 count) +// returns a record-index permutation sorted ascending by (isect, off) -- the PSI address map +// order. A stable parallel radix sort on the U64 (isect<<32|off) key; equal addresses (e.g. +// ICF-folded symbols) keep input order, which is fine for an address->symbol map. +internal U32 * +gsi_record_sort_by_sc(TP_Context *tp, Arena *arena, PDB_GsiSortRecord *arr, U64 count) { ProfBeginFunction(); - radsort(arr, count, psi_addr_map_compar_is_before); + U64 *keys = push_array_no_zero(arena, U64, count ? count : 1); + U32 *idx = push_array_no_zero(arena, U32, count ? count : 1); + for EachIndex(i, count) { + keys[i] = ((U64)arr[i].isect_off.isect << 32) | (U64)arr[i].isect_off.off; + idx[i] = (U32)i; + } + lnk_radix_sort_u64_pairs(tp, arena, count, keys, idx); ProfEnd(); + return idx; } internal @@ -1839,7 +1853,9 @@ gsi_symbol_is_before(void *raw_a, void *raw_b) } } } - is_before = cmp < 0; + // unique, element-stable tiebreaker (symbol pointer travels with the element across + // radsort swaps) prevents O(n^2) radsort on equal-key runs (ICF-folded symbols) + is_before = cmp != 0 ? (cmp < 0) : ((U64)a < (U64)b); } return is_before; @@ -1871,7 +1887,9 @@ gsi_pub_symbol_is_before(void *raw_a, void *raw_b) } } - is_before = cmp < 0; + // unique, element-stable tiebreaker (symbol pointer travels with the element across + // radsort swaps) prevents O(n^2) radsort on equal-key runs (ICF-folded publics) + is_before = cmp != 0 ? (cmp < 0) : ((U64)a < (U64)b); } return is_before; @@ -2212,15 +2230,15 @@ psi_build(TP_Context *tp, PDB_PsiContext *psi, MSF_Context *msf, MSF_StreamNumbe ProfBegin("Address Map"); ProfBegin("Sort"); - gsi_record_sort_by_sc(gsi_build.sort_record_arr, gsi_build.hash_record_count); + U32 *sorted = gsi_record_sort_by_sc(tp, scratch.arena, gsi_build.sort_record_arr, gsi_build.hash_record_count); ProfEnd(); - + ProfBegin("Offset Fill"); U64 addr_map_count = gsi_build.hash_record_count; U64 addr_map_size = addr_map_count * sizeof(U32); U32 *addr_map = push_array_no_zero(scratch.arena, U32, addr_map_count); for (U64 i = 0; i < addr_map_count; i += 1) { - addr_map[i] = gsi_build.sort_record_arr[i].offset; + addr_map[i] = gsi_build.sort_record_arr[sorted[i]].offset; } ProfEnd(); @@ -2775,12 +2793,39 @@ dbi_build_sec_con(Arena *arena, PDB_DbiContext *dbi) // sort section contribs so they are binary searchable lnk_radix_sort_dbi_sc_array(sc_array, dbi->sec_contrib_list.count, dbi->section_list.count + 1); - + + // coalesce adjacent contributions that belong to the same module, section, and + // flags and are perfectly contiguous. The section contribution substream is only + // used to map an image address range back to its module, so merging contiguous + // same-module runs is loss-less (data/reloc CRCs are unused here, always 0). UE + // objects emit one COMDAT section per function, so after layout these collapse + // heavily -- matching link.exe, which stores one entry per contiguous run. + ProfBegin("Coalesce sect contribs"); + U64 sc_count = 0; + for (U64 r = 0; r < dbi->sec_contrib_list.count; r += 1) { + PDB_DbiSectionContrib *c = &sc_array[r]; + if (sc_count > 0) { + PDB_DbiSectionContrib *p = &sc_array[sc_count - 1]; + if (p->base.sec == c->base.sec && + p->base.mod == c->base.mod && + p->base.flags == c->base.flags && + (U64)c->base.sec_off >= (U64)p->base.sec_off) { + // absorb c into the run, extending over any alignment padding gap + U64 c_end = (U64)c->base.sec_off + (U64)c->base.size; + U64 p_end = (U64)p->base.sec_off + (U64)p->base.size; + if (c_end > p_end) { p->base.size = (U32)(c_end - (U64)p->base.sec_off); } + continue; + } + } + sc_array[sc_count++] = *c; + } + ProfEnd(); + // push section contrib info ProfBegin("List Push"); String8List sec_con_list = {0}; str8_list_push(arena, &sec_con_list, str8((U8*)version, sizeof(*version))); - str8_list_push(arena, &sec_con_list, str8((U8*)sc_array, sizeof(sc_array[0])*dbi->sec_contrib_list.count)); + str8_list_push(arena, &sec_con_list, str8((U8*)sc_array, sizeof(sc_array[0])*sc_count)); ProfEnd(); ProfEnd(); diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 71003bb98..180edb169 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -68,12 +68,18 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) Semaphore exec_semaphore = {0}; if (worker_count > 1) { main_semaphore = semaphore_alloc(0, 1, str8_zero()); + // Max counts carry 2x headroom: tp_for_parallel wakes workers with a single + // batched ReleaseSemaphore(drop_count); a batch can land while up to + // worker_count-1 previously-woken workers have not yet re-taken their permit, + // so the count can transiently reach ~2*worker_count. A tight max (== worker + // count) would make that batched release exceed the max and fail outright, + // waking no one and deadlocking at the next barrier. if (is_shared) { AssertAlways(worker_count <= max_worker_count); - task_semaphore = semaphore_alloc(0, max_worker_count, name); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * max_worker_count, name); + exec_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } else { - task_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } } @@ -210,18 +216,17 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF U64 drop_count = Min(task_count, pool->worker_count); - // if we are in shared mode ping local semaphore + // Wake exactly drop_count workers in a single batched ReleaseSemaphore. The + // count MUST be drop_count (not drop_count-1): tasks that nest tp_broadcast_ + // (e.g. lnk_walk_relocs_and_mark_ref_sections_task) barrier across all + // worker_count participants, and under-waking by one leaves the barrier short + // -> deadlock. Overflow from the batch is prevented by the 2x semaphore max + // headroom set in tp_alloc. if (pool->exec_semaphore.u64[0] != 0) { - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx +=1) { - semaphore_drop(pool->exec_semaphore); - } - } - - // ping shared semaphore - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx += 1) { - semaphore_drop(pool->task_semaphore); + semaphore_drop_n(pool->exec_semaphore, (U32)drop_count); } - + semaphore_drop_n(pool->task_semaphore, (U32)drop_count); + // run tasks on main worker tp_run_tasks(pool, &pool->worker_arr[0]); diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index d96751388..bcbe6cc04 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -707,6 +707,19 @@ semaphore_drop(Semaphore semaphore) } } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + //- rjf: barriers internal Barrier diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index 799452038..35db175f1 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -692,6 +692,15 @@ semaphore_drop(Semaphore semaphore) ReleaseSemaphore(handle, 1, 0); } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + ReleaseSemaphore(handle, count, 0); + } +} + //- rjf: barriers internal Barrier