diff --git a/build.bat b/build.bat index f636ea9cd..d48ce625e 100644 --- a/build.bat +++ b/build.bat @@ -52,13 +52,15 @@ if "%pgo%"=="1" ( where llvm-profdata /q || echo llvm-profdata is not in the PATH || exit /b 1 if "%clang%"=="1" ( if "%pgo_run%" == "1" ( - call llvm-profdata merge %LLVM_PROFILE_FILE% -output=%~dp0build\build.profdata || exit /b 1 + call llvm-profdata merge %~dp0build\pgo_raw\*.profraw -output=%~dp0build\build.profdata || exit /b 1 set auto_compile_flags=%auto_compile_flags% -fprofile-use=%~dp0build\build.profdata set pgo_run=0 ) else ( echo [pgo enabled] set auto_compile_flags=%auto_compile_flags% -fprofile-generate -mllvm -vp-counters-per-site=5 - set LLVM_PROFILE_FILE=%~dp0build\build.profraw + if not exist %~dp0build\pgo_raw mkdir %~dp0build\pgo_raw + del /q %~dp0build\pgo_raw\*.profraw 2>nul + set LLVM_PROFILE_FILE=%~dp0build\pgo_raw\build.%%p.profraw set pgo_run=1 ) ) else ( @@ -142,7 +144,14 @@ pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 if "%com_shim%"=="1" set didbuild=1 && %compile% ..\src\com_shim\com_shim_main.c %compile_link% %out%com_shim.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +:: NOTE: -DBLAKE3_ATOMICS=1 makes BLAKE3 use C11 _Atomic (plain atomic load) for +:: get_cpu_features instead of MSVC's _InterlockedOr `lock or` barrier on every +:: compress dispatch (was a ~5.5s main-thread hot spot). MSVC C11 atomics require +:: /std:c11 /experimental:c11atomics. Kept external so the vendored blake3 source +:: stays pristine. +set radlink_msvc_flags= +if "%msvc%"=="1" set radlink_msvc_flags=/std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1 +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_msvc_flags% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 @@ -183,6 +192,10 @@ if "%pgo_run%"=="1" ( if "%radlink%"=="1" ( pushd local\lyra_pgo call %~dp0build\radlink @lyra.rsp || exit /b 1 + rem lyra.rsp trains with /OPT:NOREF /OPT:NOICF; run a second link with REF+ICF on + rem (later switches win) so the profile also covers the /OPT:REF walk and the ICF + rem refinement -- otherwise PGO marks them cold and the build regresses on real links + call %~dp0build\radlink @lyra.rsp /OPT:REF /OPT:ICF || exit /b 1 popd ) goto restart diff --git a/src/base/base_arena.c b/src/base/base_arena.c index 2e9b3e003..151820152 100644 --- a/src/base/base_arena.c +++ b/src/base/base_arena.c @@ -353,6 +353,54 @@ arena_pop_to(Arena *arena, U64 pos) } +//- rjf: arena decommit of unused (rewound/free) pages + +internal void +arena_decommit_unused(Arena *arena) +{ + // NOTE(perf): decommit committed-but-unused pages so they stop counting against + // working set, while keeping the reservation. Only touches pages strictly above + // the live `pos` high-water of each block in the active chain, and the unused + // bodies of free-list blocks. Live data (<= pos) is never touched. The push path + // re-commits on demand (arena_push grows `cmt`), so reuse is transparent. + if(arena->flags & ArenaFlag_LargePages) + { + // large pages cannot be partially decommitted safely; skip. + return; + } + U64 page_size = get_system_info()->page_size; + + // rjf: active chain -- decommit committed region above each block's live pos + for(Arena *n = arena->current; n != 0; n = n->prev) + { + U64 pos_aligned = AlignPow2(n->pos, page_size); + if(pos_aligned < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + pos_aligned; + U64 decommit_size = n->cmt - pos_aligned; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = pos_aligned; + } + } + +#if ARENA_FREE_LIST + // rjf: free chain -- decommit everything above the first (header) page + for(Arena *n = arena->free_last; n != 0; n = n->prev) + { + U64 keep = AlignPow2(ARENA_HEADER_SIZE, page_size); + if(keep < n->cmt) + { + U8 *decommit_ptr = (U8 *)n + keep; + U64 decommit_size = n->cmt - keep; + AsanPoisonMemoryRegion(decommit_ptr, decommit_size); + decommit_memory(decommit_ptr, decommit_size); + n->cmt = keep; + } + } +#endif +} + //- rjf: arena push/pop helpers internal void diff --git a/src/base/base_arena.h b/src/base/base_arena.h index b3ba37827..a61f5bd3f 100644 --- a/src/base/base_arena.h +++ b/src/base/base_arena.h @@ -76,6 +76,9 @@ internal void *arena_push(Arena *arena, U64 size, U64 align, B32 zero); internal U64 arena_pos(Arena *arena); internal void arena_pop_to(Arena *arena, U64 pos); +//- rjf: arena decommit of unused (rewound/free) pages +internal void arena_decommit_unused(Arena *arena); + //- rjf: arena push/pop helpers internal void arena_clear(Arena *arena); internal void arena_pop(Arena *arena, U64 amt); diff --git a/src/base/base_thread_context.c b/src/base/base_thread_context.c index 421636101..df2ec71e5 100644 --- a/src/base/base_thread_context.c +++ b/src/base/base_thread_context.c @@ -17,15 +17,21 @@ C_LINKAGE thread_static TCTX *tctx_thread_local = 0; internal TCTX * tctx_alloc(void) { + // 2MB commit quantum for scratch arenas (vs the 64KB default): scratch takes + // heavy churn on every thread (the linker pushes tens of GB through these); + // the larger quantum cuts VirtualAlloc(MEM_COMMIT) calls -- all serialized on + // the process address-space lock -- ~32x. Slack is <= 2MB per scratch arena + // past its high-water mark (2 arenas per thread), and arena_decommit_unused + // still trims page-granular, independent of the commit quantum. #if PROFILE_TELEMETRY thread_static static char name[2][1024]; raddbg_snprintf(name[0], sizeof(name[0]), "Scratch/0[TID:%u]", tid()); raddbg_snprintf(name[1], sizeof(name[1]), "Scratch/1[TID:%u]", tid()); - Arena *arena_0 = arena_alloc(.name = name[0]); - Arena *arena_1 = arena_alloc(.name = name[1]); + Arena *arena_0 = arena_alloc(.commit_size = MB(2), .name = name[0]); + Arena *arena_1 = arena_alloc(.commit_size = MB(2), .name = name[1]); #else - Arena *arena_0 = arena_alloc(); - Arena *arena_1 = arena_alloc(); + Arena *arena_0 = arena_alloc(.commit_size = MB(2)); + Arena *arena_1 = arena_alloc(.commit_size = MB(2)); #endif TCTX *tctx = push_array(arena_0, TCTX, 1); tctx->arenas[0] = arena_0; @@ -82,6 +88,21 @@ tctx_get_scratch(Arena **conflicts, U64 count) return result; } +//- rjf: scratch decommit (release committed-but-unused scratch pages back to OS) + +internal void +tctx_scratch_decommit(void) +{ + TCTX *tctx = tctx_selected(); + for(U64 i = 0; i < ArrayCount(tctx->arenas); i += 1) + { + if(tctx->arenas[i] != 0) + { + arena_decommit_unused(tctx->arenas[i]); + } + } +} + //- rjf: lane metadata internal LaneCtx diff --git a/src/base/base_thread_context.h b/src/base/base_thread_context.h index 494d1804f..e52b36476 100644 --- a/src/base/base_thread_context.h +++ b/src/base/base_thread_context.h @@ -90,6 +90,7 @@ internal TCTX *tctx_selected(void); //- rjf: scratch arenas internal Arena *tctx_get_scratch(Arena **conflicts, U64 count); +internal void tctx_scratch_decommit(void); #define scratch_begin(conflicts, count) temp_begin(tctx_get_scratch((conflicts), (count))) #define scratch_end(scratch) temp_end(scratch) diff --git a/src/base/base_threads.h b/src/base/base_threads.h index 3aae2ea6c..b40c10f9f 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -128,8 +128,12 @@ internal void semaphore_release(Semaphore semaphore); internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); +internal B32 semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us); // blocking acquire of `count` permits (off hot path) internal void semaphore_drop(Semaphore semaphore); internal void semaphore_drop_count(Semaphore semaphore, U64 drop_count); +internal void semaphore_drop_if_room(Semaphore semaphore); // best-effort post; no-op if already at max +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall +internal B32 semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out); // release 1 permit + report the pre-release count (win32: exact, from ReleaseSemaphore; posix: best-effort sem_getvalue) //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/codeview/codeview.h b/src/codeview/codeview.h index 7e4672e49..d6b3da818 100644 --- a/src/codeview/codeview.h +++ b/src/codeview/codeview.h @@ -2701,6 +2701,32 @@ struct CV_TypeIndexInfoList CV_TypeIndexInfo *last; }; +typedef struct CV_TiOff CV_TiOff; +struct CV_TiOff +{ + CV_TypeIndexSource source; + U32 offset; +}; + +// Flat, allocation-free view over a record's type-index sites. For any record kind at +// most one of the two parts is populated: +// - `arr`: fixed-shape kinds point at a static per-kind table (zero allocation); +// member-walk kinds (FIELDLIST/METHODLIST/inlinee lines) point at an arena- +// materialized array. +// - homogeneous run (count-stride kinds: ARGLIST, SUBSTR_LIST, BUILDINFO, VFTPATH, +// CALLERS/CALLEES/INLINEES): offset(i) = run_base + i*sizeof(CV_TypeIndex). +// Emission order (arr order, then ascending run) matches the legacy +// CV_TypeIndexInfoList push order exactly; hash streams depend on it. +typedef struct CV_TiOffsets CV_TiOffsets; +struct CV_TiOffsets +{ + const CV_TiOff *arr; + U32 arr_count; + CV_TypeIndexSource run_source; + U32 run_base; + U32 run_count; +}; + typedef struct CV_TypeIndexArray CV_TypeIndexArray; struct CV_TypeIndexArray { diff --git a/src/codeview/codeview_parse.c b/src/codeview/codeview_parse.c index 583666fb7..66a7c17f9 100644 --- a/src/codeview/codeview_parse.c +++ b/src/codeview/codeview_parse.c @@ -586,180 +586,269 @@ cv_symbol_type_index_info_push(Arena *arena, CV_TypeIndexInfoList *list, CV_Type SLLQueuePush(list->first, list->last, info); list->count += 1; - + return info; } -internal CV_TypeIndexInfoList -cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +//////////////////////////////// +//~ Type-index offset descriptors +// +// Static per-kind tables for fixed-shape records; homogeneous runs for +// count-stride records. Table entry order mirrors the legacy +// cv_symbol_type_index_info_push order exactly (incl. FUNC_ID IPI-before-TPI +// and UDT_SRC_LINE TPI-then-IPI asymmetries) -- leaf hash streams depend on it. + +#define CV_TIOFF(s, o) { CV_TypeIndexSource_##s, (U32)(o) } + +// leaves +read_only global CV_TiOff cv_tioffs_lf_modifier[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafModifier, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_pointer_ex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafPointer, itype)), CV_TIOFF(TPI, sizeof(CV_LeafPointer) + 0) }; +read_only global CV_TiOff cv_tioffs_lf_array[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafArray, entry_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafArray, index_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_struct2[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, field_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, derived_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafStruct2, vshape_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_union[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUnion, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_alias[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafAlias, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_func_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafFuncId, scope_string_id)), CV_TIOFF(TPI, OffsetOf(CV_LeafFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunc_id[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFuncId, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_string_id[] = { CV_TIOFF(IPI, OffsetOf(CV_LeafStringId, substr_list_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_udt_mod_src_line[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)), CV_TIOFF(IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)) }; +read_only global CV_TiOff cv_tioffs_lf_enum[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, base_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafEnum, field_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_procedure[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafProcedure, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_mfunction[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, ret_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, class_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, this_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafMFunction, arg_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vftable[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, owner_itype)), CV_TIOFF(TPI, OffsetOf(CV_LeafVFTable, base_table_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_skip[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafSkip, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_method[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMethod, list_itype)) }; +read_only global CV_TiOff cv_tioffs_lf_onemethod[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafOneMethod, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_bitfield[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafBitField, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_index[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafIndex, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_member[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafMember, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfunctab[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncTab, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_vfuncoff[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafVFuncOff, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttype[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestType, itype)) }; +read_only global CV_TiOff cv_tioffs_lf_nesttypeex[] = { CV_TIOFF(TPI, OffsetOf(CV_LeafNestTypeEx, itype)) }; + +// symbols +read_only global CV_TiOff cv_tioffs_s_buildinfo[] = { CV_TIOFF(IPI, OffsetOf(CV_SymBuildInfo, id)) }; +read_only global CV_TiOff cv_tioffs_s_data32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymData32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32_id[] = { CV_TIOFF(IPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_proc32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymProc32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_udt[] = { CV_TIOFF(TPI, OffsetOf(CV_SymUDT, itype)) }; +read_only global CV_TiOff cv_tioffs_s_thread32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymThread32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_filestatic[] = { CV_TIOFF(TPI, OffsetOf(CV_SymFileStatic, itype)) }; +read_only global CV_TiOff cv_tioffs_s_local[] = { CV_TIOFF(TPI, OffsetOf(CV_SymLocal, itype)) }; +read_only global CV_TiOff cv_tioffs_s_regrel32[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegrel32, itype)) }; +read_only global CV_TiOff cv_tioffs_s_register[] = { CV_TIOFF(TPI, OffsetOf(CV_SymRegister, itype)) }; +read_only global CV_TiOff cv_tioffs_s_constant[] = { CV_TIOFF(TPI, OffsetOf(CV_SymConstant, itype)) }; +read_only global CV_TiOff cv_tioffs_s_callsiteinfo[] = { CV_TIOFF(TPI, OffsetOf(CV_SymCallSiteInfo, itype)) }; +read_only global CV_TiOff cv_tioffs_s_inlinesite[] = { CV_TIOFF(IPI, OffsetOf(CV_SymInlineSite, inlinee)) }; +read_only global CV_TiOff cv_tioffs_s_heapallocsite[] = { CV_TIOFF(TPI, OffsetOf(CV_SymHeapAllocSite, itype)) }; + +#undef CV_TIOFF + +#define CV_TIOFFS_FIXED(result, table) do { (result).arr = (table); (result).arr_count = ArrayCount(table); } while (0) + +internal U64 +cv_ti_offsets_count(const CV_TiOffsets *offs) { - CV_TypeIndexInfoList list = {0}; + return (U64)offs->arr_count + (U64)offs->run_count; +} + +internal CV_TiOff +cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx) +{ + CV_TiOff result; + if (idx < offs->arr_count) { + result = offs->arr[idx]; + } else { + result.source = offs->run_source; + result.offset = offs->run_base + (U32)(idx - offs->arr_count) * (U32)sizeof(CV_TypeIndex); + } + return result; +} + +// grow-by-doubling scratch for member-walk kinds (FIELDLIST/METHODLIST/inlinee +// lines); memory footprint stays comparable to the legacy per-node list +typedef struct CV_TiOffBuilder +{ + Arena *arena; + CV_TiOff *v; + U32 count; + U32 cap; +} CV_TiOffBuilder; + +internal void +cv_tioff_builder_put(CV_TiOffBuilder *b, CV_TypeIndexSource source, U64 offset) +{ + if (b->count == b->cap) { + U32 new_cap = b->cap ? b->cap * 2 : 16; + CV_TiOff *new_v = push_array_no_zero(b->arena, CV_TiOff, new_cap); + MemoryCopy(new_v, b->v, sizeof(CV_TiOff) * b->count); + b->v = new_v; + b->cap = new_cap; + } + b->v[b->count].source = source; + b->v[b->count].offset = (U32)offset; + b->count += 1; +} + +internal CV_TiOffsets +cv_symbol_ti_offsets(CV_SymKind kind, String8 data) +{ + CV_TiOffsets result = {0}; switch (kind) { case CV_SymKind_BUILDINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymBuildInfo, id)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_buildinfo); } break; case CV_SymKind_GDATA32: case CV_SymKind_LDATA32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymData32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_data32); } break; case CV_SymKind_LPROC32_ID: - case CV_SymKind_GPROC32_ID: + case CV_SymKind_GPROC32_ID: case CV_SymKind_LPROC32_DPC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32_id); } break; case CV_SymKind_GPROC32: - case CV_SymKind_LPROC32: + case CV_SymKind_LPROC32: case CV_SymKind_LPROC32_DPC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymProc32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_proc32); } break; case CV_SymKind_UDT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymUDT, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_udt); } break; case CV_SymKind_GTHREAD32: case CV_SymKind_LTHREAD32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymThread32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_thread32); } break; case CV_SymKind_FILESTATIC: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymFileStatic, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_filestatic); } break; case CV_SymKind_LOCAL: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymLocal, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_local); } break; - case CV_SymKind_REGREL32: + case CV_SymKind_REGREL32: case CV_SymKind_BPREL32: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegrel32, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_regrel32); } break; case CV_SymKind_REGISTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymRegister, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_register); } break; case CV_SymKind_CONSTANT: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymConstant, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_constant); } break; case CV_SymKind_CALLSITEINFO: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymCallSiteInfo, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_callsiteinfo); } break; case CV_SymKind_CALLERS: case CV_SymKind_CALLEES: case CV_SymKind_INLINEES: { Assert(data.size >= sizeof(CV_SymFunctionList)); CV_SymFunctionList *func_list = (CV_SymFunctionList*)data.str; - for (U64 i = 0; i < func_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_SymFunctionList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_SymFunctionList); + result.run_count = func_list->count; } break; case CV_SymKind_INLINESITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_SymInlineSite, inlinee)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_inlinesite); } break; case CV_SymKind_HEAPALLOCSITE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_SymHeapAllocSite, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_s_heapallocsite); } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +internal CV_TiOffsets +cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) { - CV_TypeIndexInfoList list = {0}; + CV_TiOffsets result = {0}; switch (leaf_kind) { case CV_LeafKind_NOTYPE: case CV_LeafKind_VTSHAPE: case CV_LeafKind_LABEL: - case CV_LeafKind_NULL: + case CV_LeafKind_NULL: case CV_LeafKind_NOTTRAN: { // no type indices } break; case CV_LeafKind_MODIFIER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafModifier, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_modifier); } break; case CV_LeafKind_POINTER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafPointer, itype)); - CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; - CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); + CV_LeafPointer *ptr = (CV_LeafPointer *)data.str; + CV_PointerKind ptr_kind = CV_PointerAttribs_Extract_Kind(ptr->attribs); if (ptr_kind == CV_PointerKind_BaseType) { // TODO: add CV_LeafPointerBaseType - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); } else { CV_PointerMode ptr_mode = CV_PointerAttribs_Extract_Mode(ptr->attribs); if (ptr_mode == CV_PointerMode_PtrMem || ptr_mode == CV_PointerMode_PtrMethod) { // TODO: add type for the CvLeafPointerMember to syms_cv.mc - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafPointer) + 0); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer_ex); + } else { + CV_TIOFFS_FIXED(result, cv_tioffs_lf_pointer); } } } break; case CV_LeafKind_ARRAY: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, entry_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafArray, index_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_array); } break; - case CV_LeafKind_CLASS: + case CV_LeafKind_CLASS: case CV_LeafKind_STRUCTURE: case CV_LeafKind_INTERFACE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct); } break; case CV_LeafKind_CLASS2: case CV_LeafKind_STRUCT2: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, field_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, derived_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafStruct2, vshape_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_struct2); } break; case CV_LeafKind_UNION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUnion, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_union); } break; case CV_LeafKind_ALIAS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafAlias, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_alias); } break; case CV_LeafKind_FUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafFuncId, scope_string_id)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_func_id); } break; case CV_LeafKind_MFUNC_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFuncId, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunc_id); } break; case CV_LeafKind_STRING_ID: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafStringId, substr_list_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_string_id); } break; case CV_LeafKind_UDT_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_src_line); } break; case CV_LeafKind_UDT_MOD_SRC_LINE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafUDTModSrcLine, udt_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, OffsetOf(CV_LeafUDTModSrcLine, src_string_id)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_udt_mod_src_line); } break; case CV_LeafKind_BUILDINFO: { Assert(data.size >= sizeof(CV_LeafBuildInfo)); CV_LeafBuildInfo *build_info = (CV_LeafBuildInfo *)data.str; - for (U16 i = 0; i < build_info->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafBuildInfo) + i * sizeof(CV_ItemId)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafBuildInfo); + result.run_count = build_info->count; } break; case CV_LeafKind_ENUM: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, base_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafEnum, field_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_enum); } break; case CV_LeafKind_PROCEDURE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafProcedure, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_procedure); } break; case CV_LeafKind_MFUNCTION: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, ret_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, class_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, this_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMFunction, arg_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_mfunction); } break; case CV_LeafKind_VFTABLE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, owner_itype)); - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFTable, base_table_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vftable); } break; case CV_LeafKind_VFTPATH: { Assert(sizeof(CV_LeafVFPath) <= data.size); CV_LeafVFPath *vfpath = (CV_LeafVFPath *)data.str; - for (U32 i = 0; i < vfpath->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafVFPath) + i * sizeof(CV_TypeId)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafVFPath); + result.run_count = vfpath->count; } break; case CV_LeafKind_TYPESERVER: case CV_LeafKind_TYPESERVER2: @@ -767,76 +856,77 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data // no type indices } break; case CV_LeafKind_SKIP: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafSkip, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_skip); } break; case CV_LeafKind_SUBSTR_LIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_IPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; case CV_LeafKind_ARGLIST: { Assert(sizeof(CV_LeafArgList) <= data.size); CV_LeafArgList *arg_list = (CV_LeafArgList*)data.str; - for (U32 i = 0; i < arg_list->count; ++i) { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, sizeof(CV_LeafArgList) + i * sizeof(CV_TypeIndex)); - } + result.run_source = CV_TypeIndexSource_TPI; + result.run_base = sizeof(CV_LeafArgList); + result.run_count = arg_list->count; } break; - case CV_LeafKind_LIST: + case CV_LeafKind_LIST: case CV_LeafKind_FIELDLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { CV_LeafKind list_member_kind = 0; U64 read_size = str8_deserial_read_struct(data, cursor, &list_member_kind); - + if(read_size != sizeof(list_member_kind)) { Assert(!"malformed LF_FIELDLIST"); break; } cursor += read_size; - + switch (list_member_kind) { default: Assert(!"TODO: handle malformed field member"); break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafIndex, itype)); cursor += sizeof(CV_LeafIndex); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMember, itype)); cursor += sizeof(CV_LeafMember); - + CV_NumericParsed size; cursor += cv_read_numeric(data, cursor, &size); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_STMEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafStMember, itype)); cursor += sizeof(CV_LeafStMember); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethod, list_itype)); cursor += sizeof(CV_LeafMethod); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafOneMethod, itype)); + CV_LeafOneMethod onemethod; cursor += str8_deserial_read_struct(data, cursor, &onemethod); - + CV_MethodProp prop = CV_FieldAttribs_Extract_MethodProp(onemethod.attribs); if(prop == CV_MethodProp_PureIntro || prop == CV_MethodProp_Intro) { cursor += sizeof(U32); // virtoff } - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; @@ -849,134 +939,140 @@ cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestType, itype)); cursor += sizeof(CV_LeafNestType); - + String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafNestTypeEx, itype)); + cursor += sizeof(CV_LeafNestTypeEx); String8 name; cursor += str8_deserial_read_cstr(data, cursor, &name); } break; case CV_LeafKind_BCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafBClass, itype)); + cursor += sizeof(CV_LeafBClass); CV_NumericParsed offset; cursor += cv_read_numeric(data, cursor, &offset); } break; case CV_LeafKind_VBCLASS: case CV_LeafKind_IVBCLASS: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVBClass, itype)); cursor += sizeof(CV_LeafVBClass); - + CV_NumericParsed virtual_base_pointer; cursor += cv_read_numeric(data, cursor, &virtual_base_pointer); - + CV_NumericParsed virtual_base_offset; cursor += cv_read_numeric(data, cursor, &virtual_base_offset); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncTab, itype)); cursor += sizeof(CV_LeafVFuncTab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafVFuncOff, itype)); cursor += sizeof(CV_LeafVFuncOff); } break; } cursor = AlignPow2(cursor, 4); } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_METHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMethod, list_itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_method); } break; case CV_LeafKind_METHODLIST: { + CV_TiOffBuilder b = { arena }; for (U64 cursor = 0; cursor < data.size; ) { // read method CV_LeafMethodListMember method; U64 read_size = str8_deserial_read_struct(data, cursor, &method); - + // error check read if (read_size != sizeof(method)) { Assert(!"malformed LF_METHODLIST"); break; } - + // push type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_TPI, cursor + OffsetOf(CV_LeafMethodListMember, itype)); + // take into account intro virtual offset CV_MethodProp mprop = CV_FieldAttribs_Extract_MethodProp(method.attribs); if (mprop == CV_MethodProp_Intro || mprop == CV_MethodProp_PureIntro) { read_size += sizeof(U32); } - + // advance cursor += read_size; } + result.arr = b.v; + result.arr_count = b.count; } break; case CV_LeafKind_ONEMETHOD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafOneMethod, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_onemethod); } break; case CV_LeafKind_BITFIELD: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafBitField, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_bitfield); } break; case CV_LeafKind_PRECOMP: case CV_LeafKind_REFSYM: { // no type indices } break; case CV_LeafKind_INDEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafIndex, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_index); } break; case CV_LeafKind_MEMBER: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafMember, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_member); } break; case CV_LeafKind_VFUNCTAB: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncTab, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfunctab); } break; case CV_LeafKind_VFUNCOFF: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafVFuncOff, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_vfuncoff); } break; case CV_LeafKind_NESTTYPE: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestType, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttype); } break; case CV_LeafKind_NESTTYPEEX: { - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_TPI, OffsetOf(CV_LeafNestTypeEx, itype)); + CV_TIOFFS_FIXED(result, cv_tioffs_lf_nesttypeex); } break; default: { NotImplemented; } break; } - return list; + return result; } -internal CV_TypeIndexInfoList -cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +internal CV_TiOffsets +cv_inlinee_ti_offsets(Arena *arena, String8 raw_data) { - CV_TypeIndexInfoList list = {0}; - + CV_TiOffsets result = {0}; + CV_TiOffBuilder b = { arena }; + U64 cursor = 0; - + // first four bytes are always signature CV_C13InlineeLinesSig sig = max_U32; cursor += str8_deserial_read_struct(raw_data, cursor, &sig); - + while(cursor < raw_data.size) { // read header CV_C13InlineeSourceLineHeader *header = (CV_C13InlineeSourceLineHeader *) str8_deserial_get_raw_ptr(raw_data, cursor, sizeof(CV_C13InlineeSourceLineHeader)); - + // store type index offset - cv_symbol_type_index_info_push(arena, &list, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); - + cv_tioff_builder_put(&b, CV_TypeIndexSource_IPI, cursor + OffsetOf(CV_C13InlineeSourceLineHeader, inlinee)); + // advance past header cursor += sizeof(*header); - + // skip extra files B32 has_extra_files = (sig == CV_C13InlineeLinesSig_EXTRA_FILES); if (has_extra_files) @@ -986,10 +1082,41 @@ cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) cursor += /* file id: */ sizeof(U32) * file_count; } } - + + result.arr = b.v; + result.arr_count = b.count; + return result; +} + +internal CV_TypeIndexInfoList +cv_ti_offsets_to_list(Arena *arena, CV_TiOffsets offs) +{ + CV_TypeIndexInfoList list = {0}; + for (U64 i = 0, count = cv_ti_offsets_count(&offs); i < count; i += 1) { + CV_TiOff ti = cv_ti_offset_at(&offs, i); + cv_symbol_type_index_info_push(arena, &list, ti.source, ti.offset); + } return list; } +internal CV_TypeIndexInfoList +cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_symbol_ti_offsets(kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data) +{ + return cv_ti_offsets_to_list(arena, cv_leaf_ti_offsets(arena, leaf_kind, data)); +} + +internal CV_TypeIndexInfoList +cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data) +{ + return cv_ti_offsets_to_list(arena, cv_inlinee_ti_offsets(arena, raw_data)); +} + internal String8Array cv_get_data_around_type_indices(Arena *arena, CV_TypeIndexInfoList ti_list, String8 data) { diff --git a/src/codeview/codeview_parse.h b/src/codeview/codeview_parse.h index c33377988..b922f7eaa 100644 --- a/src/codeview/codeview_parse.h +++ b/src/codeview/codeview_parse.h @@ -291,6 +291,12 @@ internal B32 cv_is_leaf_type_server(CV_LeafKind kind); internal B32 cv_is_leaf_pch(CV_LeafKind kind); internal CV_TypeIndexSource cv_type_index_source_from_leaf_kind(CV_LeafKind leaf_kind); +internal CV_TiOffsets cv_symbol_ti_offsets(CV_SymKind kind, String8 data); // never allocates +internal CV_TiOffsets cv_leaf_ti_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); // allocates only for FIELDLIST/METHODLIST +internal CV_TiOffsets cv_inlinee_ti_offsets(Arena *arena, String8 raw_data); +internal U64 cv_ti_offsets_count(const CV_TiOffsets *offs); +internal CV_TiOff cv_ti_offset_at(const CV_TiOffsets *offs, U64 idx); + internal CV_TypeIndexInfoList cv_get_symbol_type_index_offsets(Arena *arena, CV_SymKind kind, String8 data); internal CV_TypeIndexInfoList cv_get_leaf_type_index_offsets(Arena *arena, CV_LeafKind leaf_kind, String8 data); internal CV_TypeIndexInfoList cv_get_inlinee_type_index_offsets(Arena *arena, String8 raw_data); diff --git a/src/coff/coff.c b/src/coff/coff.c index 7541f1a26..2ebe7840f 100644 --- a/src/coff/coff.c +++ b/src/coff/coff.c @@ -293,6 +293,26 @@ coff_make_import_lookup(Arena *arena, U16 hint, String8 name) return result; } +internal String8 +coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by) +{ + // IMPORT_OBJECT_NAME_NO_PREFIX: the public symbol name without a leading ?, @, or _. + // IMPORT_OBJECT_NAME_UNDECORATE: prefix skipped as above, then truncated at the first @. + String8 result = name; + if (import_by == COFF_ImportBy_NameNoPrefix || import_by == COFF_ImportBy_Undecorate) { + if (result.size > 0 && (result.str[0] == '?' || result.str[0] == '@' || result.str[0] == '_')) { + result = str8_skip(result, 1); + } + if (import_by == COFF_ImportBy_Undecorate) { + U64 at_pos = str8_find_needle(result, 0, str8_lit("@"), 0); + if (at_pos < result.size) { + result = str8_prefix(result, at_pos); + } + } + } + return result; +} + internal U32 coff_make_ordinal32(U16 hint) { diff --git a/src/coff/coff.h b/src/coff/coff.h index 5f379ee55..94b91f511 100644 --- a/src/coff/coff.h +++ b/src/coff/coff.h @@ -621,6 +621,7 @@ internal String8 coff_ordinal_data_from_hint(Arena *arena, COFF_MachineType mach internal String8 coff_make_lib_member_header(Arena *arena, String8 name, COFF_TimeStamp time_stamp, U16 user_id, U16 group_id, U16 mode, U32 size); internal String8 coff_make_import_lookup(Arena *arena, U16 hint, String8 name); +internal String8 coff_import_lookup_name_from_import_by(String8 name, COFF_ImportByType import_by); internal String8 coff_make_import_header(Arena *arena, COFF_MachineType machine, COFF_TimeStamp time_stamp, String8 dll_name, COFF_ImportByType import_by, String8 name, U16 hint_or_ordinal, COFF_ImportType type); //////////////////////////////// diff --git a/src/linker/codeview_ext/ifc.c b/src/linker/codeview_ext/ifc.c new file mode 100644 index 000000000..e5693740f --- /dev/null +++ b/src/linker/codeview_ext/ifc.c @@ -0,0 +1,107 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +read_only global U8 g_ifc_signature[4] = { 0x54, 0x51, 0x45, 0x1A }; +read_only global U8 g_uba_signature[4] = { 0x55, 0x42, 0x41, 0x01 }; // "UBA\x01" + +internal IFC_File +ifc_file_read(Arena *arena, String8 path, String8 *error_out) +{ + IFC_File ifc = {0}; + ifc.path = push_str8_copy(arena, path); + + String8 data = lnk_read_data_from_file_path(arena, 0, path); + ifc.data = data; + if (data.size < 4) { + *error_out = push_str8f(arena, "IFC '%S' is too small (%llu bytes)", path, data.size); + return ifc; + } + + // detect UBA-compressed input (out of scope) + if (MemoryMatch(data.str, g_uba_signature, sizeof(g_uba_signature))) { + *error_out = push_str8f(arena, "IFC '%S' is UBA-compressed (magic 'UBA\\x01'); materialize a raw .ifc (UBA decompress unsupported)", path); + return ifc; + } + + // validate signature + if ( ! MemoryMatch(data.str, g_ifc_signature, sizeof(g_ifc_signature))) { + *error_out = push_str8f(arena, "IFC '%S' has bad signature (expected 54 51 45 1A)", path); + return ifc; + } + + // --- parse header --- + U64 off = 4; + if (off + 32 > data.size) { goto truncated; } + MemoryCopy(ifc.content_hash, data.str + off, 32); + off += 32; + + if (off + 4 > data.size) { goto truncated; } + U8 major = data.str[off+0]; + U8 minor = data.str[off+1]; + U8 abi = data.str[off+2]; (void)abi; + U8 arch = data.str[off+3]; + off += 4; + + // assert version 0.44 + x64; error otherwise (encoding proven only for these) + if ( ! (major == 0 && minor == 44)) { + *error_out = push_str8f(arena, "IFC '%S' unsupported version %u.%u (expected 0.44)", path, major, minor); + return ifc; + } + if (arch != 2) { + *error_out = push_str8f(arena, "IFC '%S' unsupported architecture %u (expected 2 == x64)", path, arch); + return ifc; + } + + U32 cplusplus; off += str8_deserial_read_struct(data, off, &cplusplus); (void)cplusplus; + U32 string_table_bytes; off += str8_deserial_read_struct(data, off, &string_table_bytes); + U32 string_table_size; off += str8_deserial_read_struct(data, off, &string_table_size); + U32 unit; off += str8_deserial_read_struct(data, off, &unit); (void)unit; + U32 src_path; off += str8_deserial_read_struct(data, off, &src_path); (void)src_path; + U32 global_scope; off += str8_deserial_read_struct(data, off, &global_scope); (void)global_scope; + U32 toc; off += str8_deserial_read_struct(data, off, &toc); + U32 partition_count; off += str8_deserial_read_struct(data, off, &partition_count); + if (off > data.size) { goto truncated; } + + // string table + if ((U64)string_table_bytes + string_table_size > data.size) { + *error_out = push_str8f(arena, "IFC '%S' string table out of bounds", path); + return ifc; + } + String8 string_table = str8(data.str + string_table_bytes, string_table_size); + + // --- partition summary table --- + String8 needle = str8_lit(".msvc.trait.debug-records"); + U64 po = toc; + for (U32 i = 0; i < partition_count; ++i, po += 16) { + if (po + 16 > data.size) { goto truncated; } + U32 name_off, p_off, count, entity_size; + str8_deserial_read_struct(data, po + 0, &name_off); + str8_deserial_read_struct(data, po + 4, &p_off); + str8_deserial_read_struct(data, po + 8, &count); + str8_deserial_read_struct(data, po + 12, &entity_size); + + if (name_off >= string_table.size) { continue; } + String8 name = str8_cstring((char *)string_table.str + name_off); + if (str8_match(name, needle, 0)) { + // entity_size == 1 -> count is a byte length + if ((U64)p_off + count > data.size) { + *error_out = push_str8f(arena, "IFC '%S' debug-records partition out of bounds", path); + return ifc; + } + ifc.debug_records = str8(data.str + p_off, count); + break; + } + } + + if (ifc.debug_records.size == 0) { + *error_out = push_str8f(arena, "IFC '%S' has no '.msvc.trait.debug-records' partition", path); + return ifc; + } + + ifc.is_valid = 1; + return ifc; + +truncated: + *error_out = push_str8f(arena, "IFC '%S' is truncated", path); + return ifc; +} diff --git a/src/linker/codeview_ext/ifc.h b/src/linker/codeview_ext/ifc.h new file mode 100644 index 000000000..fb3169b75 --- /dev/null +++ b/src/linker/codeview_ext/ifc.h @@ -0,0 +1,43 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +#pragma once + +//////////////////////////////// +// MSVC IFC (header-unit module interface) reader +// +// radlink consumes the `.msvc.trait.debug-records` partition embedded in an +// MSVC `.ifc` (C++20 module / header-unit interface) file. That partition is a +// raw CodeView type-leaf stream (entity_size == 1, NO u32 signature, first leaf +// at offset 0, TI base 0x1000). A consuming `.obj` references this stream via +// LF_IFC_RECORD (0x1522) leaves -- see lnk_debug_info.c. +// +// File layout (microsoft/ifc-spec, FileHeader): +// u8[4] signature = { 0x54,0x51,0x45,0x1A } ("TQE\x1a") +// u8[32] content_hash (sha256) -- record.GUID(16)++record.hash(16) == first 32 bytes here +// u8 major, minor -- assert 0.44 +// u8 abi +// u8 arch -- 2 == x64 +// u32 cplusplus +// u32 string_table_bytes (off), u32 string_table_size +// u32 unit +// u32 src_path (textoffset) +// u32 global_scope +// u32 toc -- offset to partition summary table +// u32 partition_count +// u8 internal_partition +// Partition summary entry (16 bytes): { u32 name(textoffset); u32 offset; u32 count; u32 entity_size } + +typedef struct IFC_File +{ + String8 data; // whole .ifc bytes (owning view into arena) + String8 path; // .ifc path (copied) + U8 content_hash[32]; + String8 debug_records; // .msvc.trait.debug-records blob {ptr,size}; size 0 if absent + B32 is_valid; +} IFC_File; + +// Reads `path`, validates magic/version/arch, locates `.msvc.trait.debug-records`. +// On error fills *error_out and returns is_valid=0. Detects UBA-compressed inputs +// (magic "UBA\x01") and reports them (decompression is out of scope). +internal IFC_File ifc_file_read(Arena *arena, String8 path, String8 *error_out); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index be253f18a..df5fde127 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -54,6 +54,10 @@ #include "llvm/llvm.c" #include "dwarf/x64/dwarf_x64.c" +#if OS_WINDOWS +# include // GetProcessMemoryInfo for the end-of-link summary line +#endif + // --- Third Party ------------------------------------------------------------- #include "base_ext/base_blake3.h" @@ -112,6 +116,7 @@ #include "lnk_debug_helper.h" #include "lnk_obj.h" #include "lnk_lib.h" +#include "codeview_ext/ifc.h" #include "lnk_debug_info.h" #include "lnk.h" @@ -126,6 +131,7 @@ #include "lnk_obj.c" #include "lnk_debug_helper.c" #include "lnk_lib.c" +#include "codeview_ext/ifc.c" #include "lnk_debug_info.c" // ----------------------------------------------------------------------------- @@ -1426,6 +1432,7 @@ internal void lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer *inputer, LNK_SymbolTable *symtab, LNK_Link *link) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Input); Temp scratch = scratch_begin(arena->v, arena->count); U64 obj_id_base = link->objs.count; @@ -1614,6 +1621,7 @@ lnk_load_inputs(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer } scratch_end(scratch); + lnk_summary_phase_end(LNK_SummaryPhase_Input); ProfEnd(); } @@ -1806,6 +1814,14 @@ lnk_link_inputs(TP_Context *tp, ProfBeginFunction(); Temp scratch = scratch_begin(arena->v, arena->count); + // summary: this function is Input (lnk_load_inputs rounds, accumulated on its + // own bucket) + Resolve (lib search / member resolution / directives, i.e. + // everything else); attribute the remainder to Resolve at the bottom. The + // same subtraction works for every counter: they are monotonic and the Input + // brackets nest strictly inside this window + LNK_SummaryCounters summary_begin = lnk_summary_counters_now(); + LNK_SummaryCounters summary_input_at = g_summary_phase[LNK_SummaryPhase_Input]; + HashMap imports_hm = {0}; LNK_LibMemberRefList *member_ref_lists = push_array(scratch.arena, LNK_LibMemberRefList, tp->worker_count); @@ -2088,11 +2104,38 @@ lnk_link_inputs(TP_Context *tp, if (resolved_members_count == 0) { break; } } + { + LNK_SummaryCounters now = lnk_summary_counters_now(); + LNK_SummaryCounters window = lnk_summary_counters_sub_sat(now, summary_begin); + LNK_SummaryCounters input_delta = lnk_summary_counters_sub_sat(g_summary_phase[LNK_SummaryPhase_Input], summary_input_at); + LNK_SummaryCounters resolve = lnk_summary_counters_sub_sat(window, input_delta); + g_summary_phase[LNK_SummaryPhase_Resolve].wall_us += resolve.wall_us; + g_summary_phase[LNK_SummaryPhase_Resolve].user_us += resolve.user_us; + g_summary_phase[LNK_SummaryPhase_Resolve].kern_us += resolve.kern_us; + g_summary_phase[LNK_SummaryPhase_Resolve].faults += resolve.faults; + } + scratch_end(scratch); ProfEnd(); } +// Find a section's COMDAT-associative .debug$S child (per-function debug payload). Used by the +// unresolved-symbol reporter to map a reloc through the function's OWN line table. +internal U32 +lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn) +{ + String8 string_table = lnk_coff_string_table_from_obj(obj); + COFF_SectionHeader *section_table = lnk_coff_section_table_from_obj(obj); + for EachNode(assoc_n, U32Node, obj->associated_sections[fn_sn]) { + U32 sn = assoc_n->data; + if (sn == 0 || sn > obj->header.section_count_no_null) { continue; } + if (~obj->section_flags[sn-1] & LNK_SECTION_FLAG_DEBUG) { continue; } + if (str8_match(coff_name_from_section_header(string_table, §ion_table[sn-1]), str8_lit(".debug$S"), 0)) { return sn; } + } + return 0; +} + internal LNK_LinkResult lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer *inputer, LNK_SymbolTable *symtab) { @@ -2468,6 +2511,15 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer String8 section_name = coff_name_from_section_header(string_table, section_header); U64 section_number = sect_idx+1; COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + + // per-function COMDAT sections must use ONLY their associated .debug$S for line + // mapping: nothing is relocated at this point, so every fragment's LINES header + // still reads sec_off 0 -- an obj-wide accel would match apply_off against + // unrelated functions' rows and report arbitrary files/lines + CV_LineArray *section_line_arrays = 0; + U64 section_line_array_count = 0; + B32 section_lines_init = 0; + for EachIndex(reloc_idx, relocs.count) { if (supp_info.node_count > config->unresolved_symbol_ref_limit) { str8_list_pushf(scratch.arena, &supp_info, "too many unresolved symbol references reported, stopping now"); @@ -2487,21 +2539,86 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer debug_checksums = str8_list_first(&raw_checksums); debug_strings = str8_list_first(&raw_strings); } - line_matches_count = 0; - line_matches = cv_line_from_voff(debug_lines, reloc->apply_off, &line_matches_count); + if (!section_lines_init) { + section_lines_init = 1; + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, (U32)section_number); + if (child_sn) { + LNK_ObjSection child_sect = lnk_obj_section_from_sect_idx(obj, child_sn-1); + String8 child_raw = lnk_obj_get_sect_data(obj, child_sn-1, child_sect.frange); + CV_DebugS child_ds = cv_debug_s_from_data(debug_temp.arena, child_raw); + String8List frags = cv_sub_section_from_debug_s(child_ds, CV_C13SubSectionKind_Lines); + for EachNode(fn, String8Node, frags.first) { + CV_C13LinesHeaderList hl = cv_c13_lines_from_sub_sections(debug_temp.arena, fn->string, rng_1u64(0, fn->string.size)); + section_line_array_count += hl.count; + } + section_line_arrays = push_array_no_zero(debug_temp.arena, CV_LineArray, section_line_array_count ? section_line_array_count : 1); + U64 la_idx = 0; + for EachNode(fn, String8Node, frags.first) { + CV_C13LinesHeaderList hl = cv_c13_lines_from_sub_sections(debug_temp.arena, fn->string, rng_1u64(0, fn->string.size)); + for EachNode(hn, CV_C13LinesHeaderNode, hl.first) { + section_line_arrays[la_idx++] = cv_c13_line_array_from_data(debug_temp.arena, fn->string, 0, hn->v); + } + } + } + } } - if (line_matches) { - for EachIndex(i, line_matches_count) { - CV_Line line = line_matches[i]; - CV_C13Checksum checksum = {0}; - String8 file_name = {0}; - str8_deserial_read_struct(debug_checksums, line.file_off, &checksum); - str8_deserial_read_cstr(debug_strings, checksum.name_off, &file_name); - str8_list_pushf(scratch.arena, &supp_info, "%S: %S:%u", lnk_loc_from_obj(debug_temp.arena, obj), file_name, line.line_num); + // preceding-row lookup in the function's own line table: the reloc sits on the + // last source line at-or-before its offset. (cv_line_from_voff is next-row + // biased and excludes the final row's span -- unusable for this question.) + U64 printed = 0; + for EachIndex(la_idx, section_line_array_count) { + CV_LineArray *la = §ion_line_arrays[la_idx]; + if (la->line_count == 0) { continue; } + if (reloc->apply_off < la->voffs[0]) { continue; } + if (reloc->apply_off >= la->voffs[la->line_count]) { continue; } + U64 row = 0; + for (U64 r = 0; r < la->line_count; r += 1) { + if (la->voffs[r] <= reloc->apply_off) { row = r; } else { break; } + } + if (la->line_nums[row] == 0) { continue; } // compiler marker rows + CV_C13Checksum checksum = {0}; + String8 file_name = {0}; + str8_deserial_read_struct(debug_checksums, la->file_off, &checksum); + str8_deserial_read_cstr(debug_strings, checksum.name_off, &file_name); + String8 loc = push_str8f(scratch.arena, "%S: %S:%u", lnk_loc_from_obj(debug_temp.arena, obj), file_name, la->line_nums[row]); + if (supp_info.last == 0 || !str8_match(supp_info.last->string, loc, 0)) { // collapse duplicate refs to one location + str8_list_push(scratch.arena, &supp_info, loc); + } + printed += 1; + } + + // no per-function fragment covers the reloc (non-COMDAT section, or no line + // info): fall back to the obj-wide accel, else print section+offset + if (printed == 0) { + line_matches_count = 0; + line_matches = debug_lines ? cv_line_from_voff(debug_lines, reloc->apply_off, &line_matches_count) : 0; + if (section_line_array_count == 0 && line_matches) { + for EachIndex(i, line_matches_count) { + CV_Line line = line_matches[i]; + if (line.line_num == 0) { continue; } + CV_C13Checksum checksum = {0}; + String8 file_name = {0}; + str8_deserial_read_struct(debug_checksums, line.file_off, &checksum); + str8_deserial_read_cstr(debug_strings, checksum.name_off, &file_name); + String8 loc = push_str8f(scratch.arena, "%S: %S:%u", lnk_loc_from_obj(debug_temp.arena, obj), file_name, line.line_num); + if (supp_info.last == 0 || !str8_match(supp_info.last->string, loc, 0)) { + str8_list_push(scratch.arena, &supp_info, loc); + } + printed += 1; + } + } + } + if (printed == 0) { + // no line info (vftables, RTTI, data): name the section's COMDAT symbol when + // there is one -- "referenced from ??_7Foo@@6B@" beats a raw section number + LNK_ObjSymbolRef sect_symlink = {0}; + if (lnk_obj_get_comdat_symlink(obj, (U32)section_number, §_symlink)) { + COFF_ParsedSymbol sect_leader = lnk_parsed_symbol_from_coff_symbol_idx(sect_symlink.obj, sect_symlink.symbol_idx); + str8_list_pushf(scratch.arena, &supp_info, "%S: %S+%x", lnk_loc_from_obj(debug_temp.arena, obj), sect_leader.name, reloc->apply_off); + } else { + str8_list_pushf(scratch.arena, &supp_info, "%S: %S(%llx)+%x", lnk_loc_from_obj(debug_temp.arena, obj), section_name, section_number, reloc->apply_off); } - } else { - str8_list_pushf(scratch.arena, &supp_info, "%S: %S(%llx)+%x", lnk_loc_from_obj(debug_temp.arena, obj), section_name, section_number, reloc->apply_off); } } } @@ -2534,7 +2651,9 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_ref == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_summary_phase_begin(LNK_SummaryPhase_Ref); lnk_opt_ref(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Ref); } // @@ -2542,7 +2661,17 @@ lnk_link_image(TP_Context *tp, TP_Arena *arena, LNK_Config *config, LNK_Inputer // if (config->opt_icf == LNK_SwitchState_Yes) { if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } - lnk_opt_icf(tp, symtab, config, objs, link->objs.count); + lnk_summary_phase_begin(LNK_SummaryPhase_Icf); + lnk_opt_icf(tp, arena->v[0], symtab, config, objs, link->objs.count); + lnk_summary_phase_end(LNK_SummaryPhase_Icf); + } + + // + // keep line info for ICF-folded functions (bound to the leader RVA) -- see task comment + // + if (config->opt_icf == LNK_SwitchState_Yes && config->opt_ref == LNK_SwitchState_Yes) { + if (objs == 0) { objs = lnk_array_from_obj_list(scratch.arena, link->objs); } + lnk_icf_mark_folded_lines(tp, arena, objs, link->objs.count); } } @@ -2740,20 +2869,29 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) LNK_Obj **objs = task->objs; U64 objs_count = task->objs_count; - U8 **is_live = 0; - U64 *active_thread_count = 0; + // "Remove Unreachable Sections" per-task stat accumulators (reduced on task 0 for the log) + typedef struct { U64 vsize; U64 fsize; U64 section_count; } LNK_OptRefStat; + enum { LNK_OptRefStat_Null, LNK_OptRefStat_Code, LNK_OptRefStat_Data, LNK_OptRefStat_Debug, LNK_OptRefStat_Count }; + + U8 **is_live = 0; + LNK_Obj **objs_by_idx = 0; // input_idx -> obj, for the strided removal pass + U64 *active_thread_count = 0; LNK_RelocRefsBatchList *global_batch_list = 0; + LNK_OptRefStat *remove_stats = 0; if (task_id == 0) { - active_thread_count = push_array(scratch.arena, U64, 1); + remove_stats = push_array(scratch.arena, LNK_OptRefStat, LNK_OptRefStat_Count * tp->worker_count); + active_thread_count = push_array(scratch.arena, U64, 1); global_batch_list = push_array(scratch.arena, LNK_RelocRefsBatchList, 1); // alloc live flags and set live status on every non-COMDAT section - is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + is_live = push_array_no_zero(scratch.arena, U8 *, objs_count); + objs_by_idx = push_array_no_zero(scratch.arena, LNK_Obj *, objs_count ? objs_count : 1); { for EachIndex(obj_idx, objs_count) { LNK_Obj *obj = objs[obj_idx]; - is_live[obj_idx] = push_array(scratch.arena, U8, obj->header.section_count_no_null + 1); + is_live[obj_idx] = push_array(scratch.arena, U8, obj->header.section_count_no_null + 1); + objs_by_idx[obj_idx] = obj; for EachIndex(sect_idx, obj->header.section_count_no_null) { is_live[obj_idx][sect_idx + 1] = !(obj->section_flags[sect_idx] & COFF_SectionFlag_LnkCOMDAT); @@ -2813,8 +2951,21 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } } tp_broadcast(&is_live); + tp_broadcast(&objs_by_idx); tp_broadcast(&global_batch_list); tp_broadcast(&active_thread_count); + tp_broadcast(&remove_stats); + + // Per-worker memo: (obj input idx << 32 | coff symbol idx) -> final ref of the reloc-symbol + // resolve chain below. A symbol is referenced by one reloc per call site, so the chain + // (interp parse + symbol-table trie search per hop) otherwise repeats for identical inputs + // millions of times. Open-addressing and lossy (a collision past the probe window evicts); + // a miss only costs the recompute -- the cached value is a pure function of the key because + // the symbol table and parsed_symbols are read-only during /OPT:REF. + typedef struct { U64 key; LNK_Obj *obj; U64 symbol_idx; } LNK_RefResolveSlot; + U64 resolve_cache_mask = (1ull << 20) - 1; + LNK_RefResolveSlot *resolve_cache = push_array_no_zero(scratch.arena, LNK_RefResolveSlot, resolve_cache_mask + 1); + MemorySet(resolve_cache, 0xff, sizeof(resolve_cache[0]) * (resolve_cache_mask + 1)); LNK_RelocRefsBatchList free_list = {0}; for (;;) { @@ -2834,30 +2985,149 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) // reloc -> symbol LNK_ObjSymbolRef ref_symbol = (LNK_ObjSymbolRef){ .obj = batch->v[i].obj, .symbol_idx = reloc->isymbol }; - lnk_resolve_reloc_target_symbol(scratch2.arena, symtab, ref_symbol, str8_lit("/OPT:REF"), &ref_symbol); + { + // resolve-cache lookup + U64 cache_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + U64 cache_hash = cache_key * 0x9E3779B97F4A7C15ull; cache_hash ^= cache_hash >> 32; + U64 cache_slot = max_U64; + B32 cache_hit = 0; + for (U64 probe_idx = 0; probe_idx < 8; probe_idx += 1) { + U64 slot = (cache_hash + probe_idx) & resolve_cache_mask; + if (resolve_cache[slot].key == cache_key) { + ref_symbol = (LNK_ObjSymbolRef){ .obj = resolve_cache[slot].obj, .symbol_idx = (U32)resolve_cache[slot].symbol_idx }; + cache_hit = 1; + break; + } + if (resolve_cache[slot].key == max_U64) { cache_slot = slot; break; } + } + + if (!cache_hit) { + // cycle detection via linear scan of the visited chain: chains are 1-3 hops in + // practice, and this keeps the tree HashMap + per-hop arena pushes off the hot path + // (exact same first-revisit semantics) + U64 chain_fixed[64]; + U64 *chain = chain_fixed; + U64 chain_count = 0; + U64 chain_cap = ArrayCount(chain_fixed); + B32 was_cyclic = 0; + + Temp temp = temp_begin(scratch2.arena); + B32 keep_walking = 1; + do { + // detect cyclic chains + U64 symbol_key = ((U64)ref_symbol.obj->input_idx << 32ull) | (U64)ref_symbol.symbol_idx; + B32 was_seen = 0; + for EachIndex(chain_idx, chain_count) { + if (chain[chain_idx] == symbol_key) { was_seen = 1; break; } + } + if (!was_seen) { + if (chain_count == chain_cap) { + U64 *new_chain = push_array_no_zero(temp.arena, U64, chain_cap * 2); + MemoryCopy(new_chain, chain, sizeof(chain[0]) * chain_count); + chain = new_chain; chain_cap *= 2; + } + chain[chain_count++] = symbol_key; + } else { + COFF_ParsedSymbol reloc_parsed = lnk_parsed_symbol_from_coff_symbol_idx(batch->v[i].obj, reloc->isymbol); + lnk_error_obj(LNK_Warning_CyclicSymbol, batch->v[i].obj, "symbol %S forms a cyclic chain (/OPT:REF)", reloc_parsed.name); + MemoryZeroStruct(&ref_symbol); + was_cyclic = 1; + break; + } + + // unpack symbol (interp needs no name decode) + COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); + COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); + + // resolve symbol + LNK_ObjSymbolRef next_ref = {0}; + if (lnk_resolve_symbol(symtab, ref_symbol, &next_ref)) { + keep_walking = (ref_interp == COFF_SymbolValueInterp_Weak || ref_interp == COFF_SymbolValueInterp_Undefined); + ref_symbol = next_ref; + } else { + keep_walking = 0; + } + } while (keep_walking); + temp_end(temp); + + // memoize (skip the cyclic-warning path so the warning replays per reloc as before) + if (!was_cyclic) { + if (cache_slot == max_U64) { cache_slot = cache_hash & resolve_cache_mask; } + resolve_cache[cache_slot] = (LNK_RefResolveSlot){ .key = cache_key, .obj = ref_symbol.obj, .symbol_idx = ref_symbol.symbol_idx }; + } + } + } // skip unresolved symbol if (ref_symbol.obj == 0) { continue; } - // unpack resolved symbol + // unpack resolved symbol (only interp + section_number are used -- skip the name decode) COFF_ParsedSymbol ref_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref_symbol.obj, ref_symbol.symbol_idx); COFF_SymbolValueInterpType ref_interp = coff_interp_from_parsed_symbol(ref_parsed); if (ref_interp == COFF_SymbolValueInterp_Regular) { Temp temp = temp_begin(scratch2.arena); - U32List associated_sections = lnk_obj_collect_associated_sections(temp.arena, ref_symbol.obj, ref_parsed.section_number, 0); + LNK_Obj *walk_obj = ref_symbol.obj; + U32 seed_sn = ref_parsed.section_number; + + // per-walk visited set + walk stack: flat arrays with linear-scan membership -- + // associative groups are a handful of sections, and the tree HashMap + per-node + // arena pushes dominated this walk + U32 visited_fixed[64]; + U32 *visited = visited_fixed; + U64 visited_count = 0; + U64 visited_cap = ArrayCount(visited_fixed); + U32 stack_fixed[64]; + U32 *stack = stack_fixed; + U64 stack_count = 0; + U64 stack_cap = ArrayCount(stack_fixed); + stack[stack_count++] = seed_sn; + do { + U32 section_number = stack[--stack_count]; + + // detect cyclic associative sections + { + B32 was_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == section_number) { was_seen = 1; break; } + } + if (was_seen) { continue; } + if (visited_count == visited_cap) { + U32 *new_visited = push_array_no_zero(temp.arena, U32, visited_cap * 2); + MemoryCopy(new_visited, visited, sizeof(visited[0]) * visited_count); + visited = new_visited; visited_cap *= 2; + } + visited[visited_count++] = section_number; + } - // visit root section - u32_list_push(temp.arena, &associated_sections, ref_parsed.section_number); + // push associated section + for EachNode(associated_n, U32Node, walk_obj->associated_sections[section_number]) { + U32 assoc_sn = associated_n->data; - for EachNode(section_n, U32Node, associated_sections.first) { - U32 section_number = section_n->data; + + { + B32 assoc_seen = 0; + for EachIndex(visited_idx, visited_count) { + if (visited[visited_idx] == assoc_sn) { assoc_seen = 1; break; } + } + if (assoc_seen) { continue; } + } + if (stack_count == stack_cap) { + U32 *new_stack = push_array_no_zero(temp.arena, U32, stack_cap * 2); + MemoryCopy(new_stack, stack, sizeof(stack[0]) * stack_count); + stack = new_stack; stack_cap *= 2; + } + stack[stack_count++] = assoc_sn; + } COFF_SectionFlags section_flags = ref_symbol.obj->section_flags[section_number-1]; // on first section visit, set live flag and enqueue section - U8 was_visited = ins_atomic_u8_eval_assign(&is_live[ref_symbol.obj->input_idx][section_number], 1); + // (plain read first -- most targets are already live; the read keeps the flag + // cacheline shared instead of dirtying it with an unconditional exchange) + U8 was_visited = *(volatile U8 *)&is_live[walk_obj->input_idx][section_number]; + if (!was_visited) { was_visited = ins_atomic_u8_eval_assign(&is_live[walk_obj->input_idx][section_number], 1); } if (was_visited) { continue; } // is section eligible for walking? @@ -2882,7 +3152,8 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) } batch->v[batch->count++] = refs; - } + + } while (stack_count); temp_end(temp); } @@ -2915,76 +3186,63 @@ THREAD_POOL_TASK_FUNC(lnk_opt_ref_task) exit:; barrier_wait(tp->barrier); - // TODO: thread - if (task_id == 0) { + // Remove unreachable sections. Section flags are per-obj (disjoint writes), so the obj list is + // strided across tasks via objs_by_idx; stats accumulate per task and are reduced on task 0, so + // the debug log totals are identical regardless of cohort width or schedule. + { ProfBegin("Remove Unreachable Sections"); + LNK_OptRefStat *stats = remove_stats + task_id * LNK_OptRefStat_Count; + for (U64 obj_idx = task_id; obj_idx < objs_count; obj_idx += tp->worker_count) { + LNK_Obj *obj = objs_by_idx[obj_idx]; - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; for EachIndex(sect_idx, obj->header.section_count_no_null) { - U32 section_number = sect_idx+1; - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, section_number); - if ( ! is_live[obj->input_idx][section_number]) { - obj->section_flags[sect_idx] |= COFF_SectionFlag_LnkRemove; - } - } - } - - if (lnk_get_log_status(LNK_Log_Debug)) { - typedef struct { U64 vsize; U64 fsize; U64 section_count; U64 live_count; U64 live_fsize; U64 live_vsize; } Stat; - enum { Stat_Null, Stat_Code, Stat_Data, Stat_Debug, Stat_Count }; - Stat stats[Stat_Count] = {0}; - - for EachIndex(obj_idx, objs_count) { - LNK_Obj *obj = objs[obj_idx]; + U32 section_number = sect_idx+1; + if (is_live[obj->input_idx][section_number]) { continue; } - for EachIndex(sect_idx, obj->header.section_count_no_null) { - U32 section_number = sect_idx+1; - COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, section_number); + COFF_SectionHeader *section_header = lnk_coff_section_header_from_section_number(obj, section_number); + obj->section_flags[sect_idx] |= COFF_SectionFlag_LnkRemove; + COFF_SectionFlags section_flags = obj->section_flags[sect_idx]; - U64 stat_kind = Stat_Null; - if (obj->section_flags[sect_idx] & LNK_SECTION_FLAG_DEBUG) { stat_kind = Stat_Debug; } - else if (obj->section_flags[sect_idx] & COFF_SectionFlag_CntCode) { stat_kind = Stat_Code; } - else { stat_kind = Stat_Data; } + U64 stat_kind = LNK_OptRefStat_Null; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { stat_kind = LNK_OptRefStat_Debug; } + else if (section_flags & COFF_SectionFlag_CntCode) { stat_kind = LNK_OptRefStat_Code; } + else { stat_kind = LNK_OptRefStat_Data; } - if (is_live[obj->input_idx][section_number]) { - stats[stat_kind].live_count += 1; - if (obj->section_flags[sect_idx] & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].live_vsize += section_header->vsize; - } else { - stats[stat_kind].live_fsize += section_header->fsize; - } - } else { - if (obj->section_flags[sect_idx] & COFF_SectionFlag_CntUninitializedData) { - stats[stat_kind].vsize += section_header->vsize; - } else { - stats[stat_kind].fsize += section_header->fsize; - } - stats[stat_kind].section_count += 1; - } + if (section_flags & COFF_SectionFlag_CntUninitializedData) { + stats[stat_kind].vsize += section_header->vsize; + } else { + stats[stat_kind].fsize += section_header->fsize; } + stats[stat_kind].section_count += 1; } + } + ProfEnd(); + } + barrier_wait(tp->barrier); - U64 total_fsize = 0, total_section_count = 0; - U64 total_fsize_live = 0, total_section_count_live = 0; - for EachElement(i, stats) { - total_fsize += stats[i].fsize; - total_section_count += stats[i].section_count; - total_fsize_live += stats[i].live_fsize; - total_section_count_live += stats[i].live_count; + if (task_id == 0 && lnk_get_log_status(LNK_Log_Debug)) { + LNK_OptRefStat stats[LNK_OptRefStat_Count] = {0}; + for EachIndex(reduce_task_idx, tp->worker_count) { + for EachIndex(stat_idx, (U64)LNK_OptRefStat_Count) { + stats[stat_idx].vsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].vsize; + stats[stat_idx].fsize += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].fsize; + stats[stat_idx].section_count += remove_stats[reduce_task_idx * LNK_OptRefStat_Count + stat_idx].section_count; } - String8List stat_list = {0}; - str8_list_pushf(scratch.arena, &stat_list, "Code : removed %M, %S sections; live %M, %S sections", stats[Stat_Code].fsize, str8_from_count(scratch.arena, stats[Stat_Code].section_count ), stats[Stat_Code].live_fsize, str8_from_count(scratch.arena, stats[Stat_Code].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Data : removed %M, %S sections; live %M, %S sections", stats[Stat_Data].fsize, str8_from_count(scratch.arena, stats[Stat_Data].section_count ), stats[Stat_Data].live_fsize, str8_from_count(scratch.arena, stats[Stat_Data].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Debug: removed %M, %S sections; live %M, %S sections", stats[Stat_Debug].fsize, str8_from_count(scratch.arena, stats[Stat_Debug].section_count), stats[Stat_Debug].live_fsize, str8_from_count(scratch.arena, stats[Stat_Debug].live_count)); - str8_list_pushf(scratch.arena, &stat_list, "Total: removed %M, %S sections; live %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count), total_fsize_live, str8_from_count(scratch.arena, total_section_count_live)); - String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); - lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - ProfEnd(); + U64 total_fsize = 0, total_section_count = 0; + for EachElement(i, stats) { + total_fsize += stats[i].fsize; + total_section_count += stats[i].section_count; + } + String8List stat_list = {0}; + str8_list_pushf(scratch.arena, &stat_list, "Code : %M, %S sections", stats[LNK_OptRefStat_Code].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Code].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Data : %M, %S sections", stats[LNK_OptRefStat_Data].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Data].section_count )); + str8_list_pushf(scratch.arena, &stat_list, "Debug: %M, %S sections", stats[LNK_OptRefStat_Debug].fsize, str8_from_count(scratch.arena, stats[LNK_OptRefStat_Debug].section_count)); + str8_list_pushf(scratch.arena, &stat_list, "Total: %M, %S sections", total_fsize, str8_from_count(scratch.arena, total_section_count)); + String8 stat_str = str8_list_join(scratch.arena, &stat_list, &(StringJoin){.pre = str8_lit(" "), .sep = str8_lit("\n ")}); + lnk_log(LNK_Log_Debug, "/OPT:REF Stats:\n%S", stat_str); } - barrier_wait(tp->barrier); scratch_end(scratch2); scratch_end(scratch); @@ -2996,9 +3254,15 @@ lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj { ProfBegin("/OPT:REF"); Temp scratch = scratch_begin(0,0); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_ref_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_ref_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); } @@ -3253,6 +3517,10 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U64 *split_offsets = 0; U32 *is_part_stable = 0; U64 *next_color = 0; + U64 *last_split_rng = 0; // (lo, hi]: colors allocated by the PREVIOUS round's splits. + // next_color is monotone and a split's retained subgroup keeps its + // exact old color value, so a target's color changed last round + // IFF it lies in this window -- the whole dirty test is one read. if (task_id == 0) { ProfBegin("Init"); @@ -3276,6 +3544,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) split_offsets = push_array(scratch.arena, U64, tp->worker_count + 1); is_part_stable = push_array(scratch.arena, U32, 1); next_color = push_array(scratch.arena, U64, 1); + last_split_rng = push_array(scratch.arena, U64, 2); *next_color = LNK_ICF_ColorSpace_COUNT + noncontrib_count; lnk_log(LNK_Log_Debug, " Contrib count: %S", str8_from_count(scratch.arena, contrib_count)); @@ -3291,6 +3560,7 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) tp_broadcast(&split_offsets); tp_broadcast(&is_part_stable); tp_broadcast(&next_color); + tp_broadcast(&last_split_rng); ProfBegin("Compute Hashes"); HashMap reloc_target_hm = {0}; // cache source-symbol resolution so refinement only reads colors and hashes @@ -3452,22 +3722,58 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) U64 ready_state = (table_generation_value << 2) | 2; ProfBegin("Compute colored hashes"); + B32 use_xxh3 = task->config->icf_hash_xxh3; for EachInRange(contrib_idx, contrib_ranges[task_id]) { Contrib *contrib = &contribs[contrib_idx]; contrib->key.old_color = color_map[contrib->obj_idx][contrib->sect_idx]; - blake3_hasher hasher; blake3_hasher_init(&hasher); - blake3_hasher_update(&hasher, &contrib->static_hash, sizeof(contrib->static_hash)); - for EachIndex(reloc_idx, contrib->reloc_count) { - RelocTarget *target = contrib->reloc_targets[reloc_idx]; - U64 target_id = target->color ? *target->color : target->static_id; - blake3_hasher_update(&hasher, &target_id, sizeof(target_id)); + // the key hash is a pure function of static_hash + target colors, so recompute it only + // when one of this contrib's target colors changed in the previous round's update -- + // unchanged inputs reproduce the cached hash bit-for-bit. A color changed last round + // IFF its value lies in the previous round's split-allocation window (colors are + // monotone and a split's retained subgroup keeps its exact old value), so the dirty + // test is one read per target. Refinement converges fast, so after the first few + // rounds almost every contrib skips the hash entirely. + B32 must_hash = (iter_count == 0); + if (!must_hash) { + U64 dirty_lo = last_split_rng[0], dirty_hi = last_split_rng[1]; + for EachIndex(reloc_idx, contrib->reloc_count) { + RelocTarget *target = contrib->reloc_targets[reloc_idx]; + if (target->color) { + U64 c = *target->color; + if (c > dirty_lo && c <= dirty_hi) { must_hash = 1; break; } + } + } + } + if (must_hash) { + if (use_xxh3) { + // /RAD_ICF_HASH_ALG:XXH3 -- non-cryptographic round keys; group equality still + // compares the full 128-bit hash + old_color, and the content identity that decides + // WHAT folds stays anchored by the blake3 static_hash mixed into every key + XXH3_state_t hasher; XXH3_128bits_reset(&hasher); + XXH3_128bits_update(&hasher, &contrib->static_hash, sizeof(contrib->static_hash)); + for EachIndex(reloc_idx, contrib->reloc_count) { + RelocTarget *target = contrib->reloc_targets[reloc_idx]; + U64 target_id = target->color ? *target->color : target->static_id; + XXH3_128bits_update(&hasher, &target_id, sizeof(target_id)); + } + XXH128_hash_t hash = XXH3_128bits_digest(&hasher); + MemoryCopy(&contrib->key.hash, &hash, sizeof(contrib->key.hash)); + } else { + blake3_hasher hasher; blake3_hasher_init(&hasher); + blake3_hasher_update(&hasher, &contrib->static_hash, sizeof(contrib->static_hash)); + for EachIndex(reloc_idx, contrib->reloc_count) { + RelocTarget *target = contrib->reloc_targets[reloc_idx]; + U64 target_id = target->color ? *target->color : target->static_id; + blake3_hasher_update(&hasher, &target_id, sizeof(target_id)); + } + U128 hash; + blake3_hasher_finalize(&hasher, (U8 *)&hash, sizeof(hash)); + contrib->key.hash = hash; + } } - U128 hash; - blake3_hasher_finalize(&hasher, (U8 *)&hash, sizeof(hash)); // insert the colored hash into the concurrent table - contrib->key.hash = hash; Assert(color_table.slots_count > 0 && (color_table.slots_count & (color_table.slots_count - 1)) == 0); U64 table_hash = hash_map_hasher(str8_struct(&contrib->key)); @@ -3579,10 +3885,15 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) total_split_count += split_counts[worker_id]; } split_offsets[tp->worker_count] = start_next_color; - + *next_color += total_split_count; *is_part_stable = (total_split_count == 0); + // publish this round's split-allocation window for the next round's dirty test + // (fresh colors handed out below are start_next_color+1 .. *next_color) + last_split_rng[0] = start_next_color; + last_split_rng[1] = *next_color; + lnk_log(LNK_Log_Debug, " Round %llu found %S splits", iter_count, str8_from_count(scratch.arena, total_split_count)); ProfEnd(); @@ -3674,6 +3985,13 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) contrib_obj->symlinks[contrib->sect_idx + 1] = (LNK_ObjSymbolRef){ leader_obj, leader_obj->comdats[leader->sect_idx] }; contrib_obj->section_flags[contrib->sect_idx] |= COFF_SectionFlag_LnkRemove; + // record the fold for debug aliasing (lnk_icf_mark_folded_lines): unlike the symlink + // redirect, this distinguishes an ICF fold (different-named section joined to a leader) + // from same-name COMDAT selection and /OPT:REF removal + if (contrib_obj->icf_fold) { + contrib_obj->icf_fold[contrib->sect_idx] = (LNK_ICFFold){ .leader_obj_idx = (U32)leader->obj_idx, .leader_sn = (U32)(leader->sect_idx + 1), .set = 1 }; + } + #if LNK_PARANOID String8 section_name = lnk_obj_section_name_from_section_number(contrib_obj, contrib->sect_idx+1); String8 leader_name = lnk_obj_section_name_from_section_number(leader_obj, leader->sect_idx+1); @@ -3752,15 +4070,32 @@ THREAD_POOL_TASK_FUNC(lnk_opt_icf_task) } internal void -lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) +lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count) { ProfBegin("/OPT:ICF"); - Temp scratch = scratch_begin(0,0); - + Temp scratch = scratch_begin(&perm, 1); + lnk_log(LNK_Log_Debug, "/OPT:ICF:"); - U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, tp->worker_count, objs, objs_count); + + // per-section fold map, consumed by the debug-aliasing pass after /OPT:REF + // (lnk_icf_mark_folded_lines); allocated only when that pass will run + if (config->opt_ref == LNK_SwitchState_Yes) { + ProfScope("Alloc fold maps") { + for EachIndex(obj_idx, objs_count) { + objs[obj_idx]->icf_fold = push_array(perm, LNK_ICFFold, objs[obj_idx]->header.section_count_no_null); + } + } + } + + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier)/tp_broadcast, + // so under /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- + // a plain tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane obj distribution so both agree on the width. + U32 C = tp_barrier_begin(tp); + U32Array *obj_indices = lnk_obj_indices_from_section_counts(scratch.arena, C, objs, objs_count); LNK_OptTask task = { .symtab = symtab, .config = config, .objs = objs, .objs_count = objs_count, .obj_indices = obj_indices }; - tp_for_parallel(tp, 0, tp->worker_count, lnk_opt_icf_task, &task); + tp_for_parallel_reserve(tp, 0, C, lnk_opt_icf_task, &task); // BARRIER pass (path B) + tp_barrier_end(tp); scratch_end(scratch); ProfEnd(); @@ -3808,6 +4143,204 @@ lnk_should_gather_section(LNK_Obj *obj, U64 sect_idx, COFF_SectionHeader *sect_h return 1; } +typedef struct +{ + LNK_Obj **objs; // indexed by input_idx (== task_id) +} LNK_ICFMarkFoldedLinesTask; + + +// FILECHKSMS of the obj-wide (non-COMDAT) .debug$S -- the table every per-function +// Lines fragment's file_off indexes into. Direct header walk with early-out instead of +// cv_debug_s_from_data: the obj-wide .debug$S is megabytes of subsections and the full +// parse pushes a list node per subsection; here we only need one slice. +internal String8 +lnk_icf_obj_file_chksms_scan(LNK_Obj *obj) +{ + String8 string_table = lnk_coff_string_table_from_obj(obj); + COFF_SectionHeader *section_table = lnk_coff_section_table_from_obj(obj); + for EachIndex(sect_idx, obj->header.section_count_no_null) { + if (~obj->section_flags[sect_idx] & LNK_SECTION_FLAG_DEBUG) { continue; } + if (obj->section_flags[sect_idx] & COFF_SectionFlag_LnkCOMDAT) { continue; } + if (!str8_match(coff_name_from_section_header(string_table, §ion_table[sect_idx]), str8_lit(".debug$S"), 0)) { continue; } + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, sect_idx); + String8 raw = lnk_obj_get_sect_data(obj, sect_idx, sect.frange); + if (raw.size < sizeof(CV_Signature) || cv_signature_from_debug_s(raw) != CV_Signature_C13) { continue; } + for (U64 cursor = sizeof(CV_Signature); cursor + sizeof(CV_C13SubSectionHeader) <= raw.size; ) { + CV_C13SubSectionHeader header = {0}; + cursor += str8_deserial_read_struct(raw, cursor, &header); + if (header.kind == CV_C13SubSectionKind_FileChksms) { + return str8_substr(raw, r1u64(cursor, cursor + header.size)); + } + cursor += header.size; + cursor = AlignPow2(cursor, CV_C13SubSectionAlign); + } + } + return str8_zero(); +} + +// Memoized per obj: leaders are shared across many follower objs, so without the memo the +// scan reruns once per (follower obj x leader switch). The result slices the immutable +// obj->data mapping, so the racy fill is idempotent (every worker writes identical bytes); +// the init flag is published last. +internal String8 +lnk_icf_obj_file_chksms(LNK_Obj *obj) +{ + if (!ins_atomic_u32_eval((U32 *)&obj->icf_file_chksms_init)) { + String8 chksms = lnk_icf_obj_file_chksms_scan(obj); + obj->icf_file_chksms = chksms; + ins_atomic_u32_eval_assign((U32 *)&obj->icf_file_chksms_init, 1); + } + return obj->icf_file_chksms; +} + +// source identity of a function: (checksum of its file, first line). Two ICF fold members +// with equal keys are the same source text (template twins) -- their locals/labels are +// identical and the leader's record tree serves both. +typedef struct +{ + B32 valid; + U32 line; + U8 chksum_kind; + String8 chksum; +} LNK_ICFSrcKey; + +internal LNK_ICFSrcKey +lnk_icf_src_key_from_fn(Arena *scratch, LNK_Obj *obj, U32 fn_sn, String8 chksms) +{ + LNK_ICFSrcKey key = {0}; + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, fn_sn); + if (child_sn == 0 || chksms.size == 0) { return key; } + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, child_sn-1); + String8 raw = lnk_obj_get_sect_data(obj, child_sn-1, sect.frange); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + if (lines.node_count == 0) { return key; } + String8 frag = lines.first->string; + if (frag.size < sizeof(CV_C13SubSecLinesHeader) + sizeof(CV_C13File) + sizeof(CV_C13Line)) { return key; } + CV_C13File *file = (CV_C13File *)(frag.str + sizeof(CV_C13SubSecLinesHeader)); + CV_C13Line *l0 = (CV_C13Line *)((U8 *)file + sizeof(CV_C13File)); + if ((U64)file->file_off + sizeof(CV_C13Checksum) > chksms.size) { return key; } + CV_C13Checksum *ck = (CV_C13Checksum *)(chksms.str + file->file_off); + if ((U64)file->file_off + sizeof(CV_C13Checksum) + ck->len > chksms.size) { return key; } + key.valid = 1; + key.line = (U32)(l0->flags & 0xFFFFFF); + key.chksum_kind = ck->kind; + key.chksum = str8(chksms.str + file->file_off + sizeof(CV_C13Checksum), ck->len); + return key; +} + +// does the function's record tree have anything a watch window would show? +internal B32 +lnk_icf_debug_s_has_locals(Arena *scratch, LNK_Obj *obj, U32 child_sn) +{ + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, child_sn-1); + String8 raw = lnk_obj_get_sect_data(obj, child_sn-1, sect.frange); + CV_DebugS ds = cv_debug_s_from_data(scratch, raw); + String8List syms = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Symbols); + for EachNode(n, String8Node, syms.first) { + String8 s = n->string; + for (U64 o = 0; o + 4 <= s.size; ) { + U16 len, kind; + MemoryCopy(&len, s.str + o, sizeof(len)); + MemoryCopy(&kind, s.str + o + 2, sizeof(kind)); + if (len < 2) { break; } + switch (kind) { + // stack locals + case CV_SymKind_LOCAL: + case CV_SymKind_REGREL32: + // function-scoped statics (S_LDATA32 and friends): the record naming the static lives in + // this tree; if it were dropped the debugger could no longer evaluate the follower's + // static by name, even though the (folded) data itself survives in the image + case CV_SymKind_LDATA32: + case CV_SymKind_GDATA32: + case CV_SymKind_LTHREAD32: + case CV_SymKind_GTHREAD32: + case CV_SymKind_FILESTATIC: + case CV_SymKind_CONSTANT: + return 1; + } + o += len + 2; + } + } + return 0; +} + +// /OPT:ICF folded-function debug-info slimming. Without this pass a folded function's associated +// .debug$S stays collected in full (REF marked it live with its then-live parent; the ICF fold +// removes only the .text follower), so the module stream receives every folded body's WHOLE record +// tree (S_GPROC/locals + Lines) bound to the leader RVA -- link.exe-parity content at ~+30% module +// bytes. The C13 Lines subsections are ~1% of that and are all a source breakpoint needs to bind. +// So mark each folded follower's associated .debug$S LnkRemove (drops it from full collection); +// the reloc patcher still patches it and the C13 pass merges back ONLY its Lines -- their +// SECREL/SECTION relocs target the folded function symbol, which resolves to the leader RVA +// through the redirected symlink/sect_map, so the lines land on the surviving body. +internal +THREAD_POOL_TASK_FUNC(lnk_icf_mark_folded_lines_task) +{ + Temp scratch = scratch_begin(&arena, 1); + + LNK_ICFMarkFoldedLinesTask *task = raw_task; + LNK_Obj *obj = task->objs[task_id]; + if (obj->icf_fold == 0) { scratch_end(scratch); return; } + + for EachIndex(sect_idx, obj->header.section_count_no_null) { + LNK_ICFFold fold = obj->icf_fold[sect_idx]; + if (!fold.set) { continue; } + if (~obj->section_flags[sect_idx] & COFF_SectionFlag_LnkRemove) { continue; } // follower kept live -> its own records emit + LNK_Obj *leader_obj = task->objs[fold.leader_obj_idx]; + if (leader_obj->section_flags[fold.leader_sn - 1] & COFF_SectionFlag_LnkRemove) { continue; } // whole class dead-stripped + + // Lines-only by default. Escalate to the FULL record tree (link.exe parity for this one + // fold) when the follower comes from a DIFFERENT source location than the leader (else the + // trees are textually identical -- template twins) AND it has locals to show. Measured on + // the FN editor DLL: ~6.5% of folds differ in source, most of those are empty virtuals, so + // the escalation set is small. + U8 mark = 1; + { + Temp fold_temp = temp_begin(scratch.arena); + U32 child_sn = lnk_icf_debug_s_child_from_section(obj, (U32)(sect_idx + 1)); + if (child_sn != 0) { + String8 follower_chksms = lnk_icf_obj_file_chksms(obj); // per-obj memo -- leaders + String8 leader_chksms = lnk_icf_obj_file_chksms(leader_obj); // shared across follower objs + LNK_ICFSrcKey fk = lnk_icf_src_key_from_fn(fold_temp.arena, obj, (U32)(sect_idx + 1), follower_chksms); + LNK_ICFSrcKey lk = lnk_icf_src_key_from_fn(fold_temp.arena, leader_obj, fold.leader_sn, leader_chksms); + B32 same_src = fk.valid && lk.valid && + fk.line == lk.line && fk.chksum_kind == lk.chksum_kind && + str8_match(fk.chksum, lk.chksum, 0); + if (fk.valid && lk.valid && !same_src && lnk_icf_debug_s_has_locals(fold_temp.arena, obj, child_sn)) { + mark = 2; + } + } + temp_end(fold_temp); + } + + for EachNode(assoc_n, U32Node, obj->associated_sections[sect_idx + 1]) { + U32 assoc_sn = assoc_n->data; + if (assoc_sn == 0 || assoc_sn > obj->header.section_count_no_null) { continue; } + if (~obj->section_flags[assoc_sn - 1] & LNK_SECTION_FLAG_DEBUG) { continue; } + // exclude the follower's .debug$S from full module collection (it would otherwise merge + // its whole record tree at the leader RVA, link.exe-parity size); the consumers below + // merge back just its Lines (mark 1) or, rarely, the whole tree (mark 2) + obj->section_flags[assoc_sn - 1] |= COFF_SectionFlag_LnkRemove; + if (obj->icf_lines_only == 0) { + obj->icf_lines_only = push_array(arena, B8, obj->header.section_count_no_null); + } + obj->icf_lines_only[assoc_sn - 1] = mark; + } + } + + scratch_end(scratch); +} + +internal void +lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count) +{ + ProfBeginFunction(); + LNK_ICFMarkFoldedLinesTask task = { .objs = objs }; + tp_for_parallel(tp, arena, objs_count, lnk_icf_mark_folded_lines_task, &task); // arena: per-obj icf_lines_only bitmaps + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_gather_sections_task) { @@ -4022,6 +4555,21 @@ THREAD_POOL_TASK_FUNC(lnk_set_comdat_leaders_contribs_task) ProfEnd(); } +internal +THREAD_POOL_TASK_FUNC(lnk_copy_symbol_tables_task) +{ + LNK_BuildImageTask *task = raw_task; + U64 obj_idx = task_id; + LNK_Obj *obj = task->objs[obj_idx]; + + U64 copy_size = task->u.patch_symtabs.symtab_copy_offsets[obj_idx+1] - task->u.patch_symtabs.symtab_copy_offsets[obj_idx]; + if (copy_size > 0) { + U8 *copy = task->u.patch_symtabs.symtab_copy_base + task->u.patch_symtabs.symtab_copy_offsets[obj_idx]; + MemoryCopy(copy, obj->data.str + obj->header.symbol_table_range.min, copy_size); + obj->symbol_table_copy = str8(copy, copy_size); + } +} + internal THREAD_POOL_TASK_FUNC(lnk_flag_debug_symbols_task) { @@ -4432,6 +4980,48 @@ THREAD_POOL_TASK_FUNC(lnk_patch_weak_symbols_task) lnk_patch_obj_symtab(task->symtab, task->objs[task_id], task->u.patch_symtabs.was_symbol_patched[task_id], COFF_SymbolValueInterp_Weak); } +// Non-temporal (streaming) stores for the write-once image buffer. The image is +// filled, then streamed straight to disk; these bytes are not re-read by the +// filling thread, so NT stores avoid polluting L2/L3 with ~1GB of write-once +// data. A later pass (lnk_obj_reloc_patcher) DOES read the image back, so every +// caller must _mm_sfence() before that pass runs to make the NT stores globally +// visible. NT stores require 32B alignment; the unaligned head/tail and small +// (<256B) copies fall back to MemoryCopy/MemorySet (identical bytes either way). +#define LNK_STREAM_MIN_SIZE 256 + +// SSE2 (baseline on x86-64, no -mavx required) 16B non-temporal stores. +internal void +lnk_stream_copy(void *dst, void *src, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemoryCopy(dst, src, size); return; } + U8 *d = (U8 *)dst, *s = (U8 *)src; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; // bytes to reach 16B-aligned dst + if (head) { MemoryCopy(d, s, head); d += head; s += head; size -= head; } + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + __m128i v = _mm_loadu_si128((__m128i const *)(s + i)); + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemoryCopy(d + vec, s + vec, tail); } +} + +internal void +lnk_stream_set(void *dst, U8 byte, U64 size) +{ + if (size < LNK_STREAM_MIN_SIZE) { MemorySet(dst, byte, size); return; } + U8 *d = (U8 *)dst; + U64 head = (U64)(0x10 - ((U64)d & 0xf)) & 0xf; + if (head) { MemorySet(d, byte, head); d += head; size -= head; } + __m128i v = _mm_set1_epi8((char)byte); + U64 vec = size & ~(U64)0xf; + for (U64 i = 0; i < vec; i += 0x10) { + _mm_stream_si128((__m128i *)(d + i), v); + } + U64 tail = size - vec; + if (tail) { MemorySet(d + vec, byte, tail); } +} + internal THREAD_POOL_TASK_FUNC(lnk_image_fill_task) { @@ -4441,15 +5031,45 @@ THREAD_POOL_TASK_FUNC(lnk_image_fill_task) for EachNode(n, LNK_ImageFillNode, task->u.image_fill.fill_nodes[task_id]) { for EachIndex(i, n->sc_count) { LNK_SectionContrib *sc = n->sc[i]; + // fast-path: the vast majority of contribs are a single data-node -> one direct copy, skipping + // the list-walk + cursor bookkeeping on the hot 739MB image-write loop. + if (sc->first_data_node.next == 0) { + U64 image_off = sc->u.off + n->base_foff; + Assert(image_off + sc->first_data_node.string.size <= image_data.size); + lnk_stream_copy(image_data.str + image_off, sc->first_data_node.string.str, sc->first_data_node.string.size); + continue; + } U64 cursor = 0; for EachNode(data_n, String8Node, &sc->first_data_node) { U64 image_off = sc->u.off + n->base_foff + cursor; Assert(image_off + data_n->string.size <= image_data.size); - MemoryCopyStr8(image_data.str + image_off, data_n->string); + lnk_stream_copy(image_data.str + image_off, data_n->string.str, data_n->string.size); cursor += data_n->string.size; } } } + // NT stores above are not ordered wrt later normal reads on other cores; the + // reloc-patch pass reads the image back. Make these stores globally visible. + _mm_sfence(); + ProfEnd(); +} + +typedef struct +{ + U8 *dst; + U64 size; + U8 byte; +} LNK_FillAlignRange; + +internal +THREAD_POOL_TASK_FUNC(lnk_fill_align_bytes_task) +{ + ProfBeginFunction(); + LNK_FillAlignRange *range = &((LNK_FillAlignRange *)raw_task)[task_id]; + lnk_stream_set(range->dst, range->byte, range->size); + // make this task's NT stores globally visible before the completion counter is + // bumped, so every thread past the join (contrib-fill / reloc passes) sees them + _mm_sfence(); ProfEnd(); } @@ -4466,10 +5086,34 @@ lnk_compute_win32_image_header_size(LNK_Config *config, U64 sect_count) return image_header_size; } +// reloc patch ordering: sort each section's relocs by apply_off so the RMW +// write stream into the image is monotone-forward (HW-prefetchable) instead of +// scattered in on-disk table order. apply_off is the primary key; orig_idx is a +// tiebreak so the total order is deterministic and the final bytes are identical +// to the unsorted patch order (each reloc writes its own disjoint field; only a +// pathological same-apply_off overlap could depend on order, and the orig_idx +// tiebreak preserves the original sequence there too). +typedef struct LNK_RelocSortKey +{ + COFF_Reloc reloc; + U32 orig_idx; +} LNK_RelocSortKey; + +internal int +lnk_reloc_sort_key_is_before(void *raw_a, void *raw_b) +{ + LNK_RelocSortKey *a = raw_a, *b = raw_b; + if (a->reloc.apply_off != b->reloc.apply_off) { + return a->reloc.apply_off < b->reloc.apply_off; + } + return a->orig_idx < b->orig_idx; +} + internal THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) { ProfBeginFunction(); + Temp scratch = scratch_begin(0, 0); LNK_ObjRelocPatcher *task = raw_task; LNK_Obj *obj = task->objs[task_id]; @@ -4479,27 +5123,56 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) String8 symbol_table = lnk_coff_symbol_table_from_obj(obj); String8 string_table = lnk_coff_string_table_from_obj(obj); - U32 closest_sect = 0; - U32 closest_reloc = 0; - U32 closest_foff = max_U32; - for EachIndex(sect_idx, obj_header.section_count_no_null) { COFF_SectionHeader *section_header = §ion_table[sect_idx]; COFF_SectionFlags section_flags = obj->section_flags[sect_idx]; if (section_flags & COFF_SectionFlag_LnkInfo) { continue; } - if (section_flags & COFF_SectionFlag_LnkRemove) { continue; } + if (section_flags & COFF_SectionFlag_LnkRemove) { + // exception: ICF-folded functions' .debug$S stays dead but its Lines are merged into the + // module bound to the leader RVA -- patch it so those Lines carry real addresses + if (!(obj->icf_lines_only && obj->icf_lines_only[sect_idx])) { continue; } + } if (section_flags & COFF_SectionFlag_CntUninitializedData) { continue; } + COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + // get section bytes (special case debug info because it is not copied to the image) - String8 data = section_flags & LNK_SECTION_FLAG_DEBUG ? obj->data : task->image_data; Rng1U64 section_frange = rng_1u64(section_header->foff, section_header->foff + section_header->fsize); - String8 section_data = str8_substr(data, section_frange); + String8 section_data; + if (section_flags & LNK_SECTION_FLAG_DEBUG) { + // debug sections only feed the PDB/RDI path; objs excluded from debug info never get + // there, so patching their debug relocs would only dirty input pages for nothing + if (obj->exclude_from_debug_info) { continue; } + + // nothing to patch -- readers consume the (clean) input view directly + if (relocs.count == 0) { continue; } + + // patch-on-copy: relocs on debug sections would otherwise dirty the copy-on-write input + // mapping (input views are FILE_MAP_COPY); copy the section into private memory and patch + // the copy. Readers pick the copy up through lnk_obj_get_sect_data. + if (obj->sect_data_copies == 0) { + obj->sect_data_copies = push_array(arena, String8, obj_header.section_count_no_null); + } + String8 src = str8_substr(obj->data, section_frange); + U8 *copy = push_array_no_zero(arena, U8, src.size); + MemoryCopy(copy, src.str, src.size); + obj->sect_data_copies[sect_idx] = str8(copy, src.size); + section_data = obj->sect_data_copies[sect_idx]; + } else { + section_data = str8_substr(task->image_data, section_frange); + } - // apply relocs - COFF_RelocArray relocs = lnk_coff_relocs_from_section_header(obj, section_header); + // apply relocs (sorted by apply_off for monotone-forward image writes) + Temp reloc_temp = temp_begin(scratch.arena); + LNK_RelocSortKey *sorted_relocs = push_array_no_zero(reloc_temp.arena, LNK_RelocSortKey, relocs.count); for EachIndex(reloc_idx, relocs.count) { - COFF_Reloc *reloc = &relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].reloc = relocs.v[reloc_idx]; + sorted_relocs[reloc_idx].orig_idx = (U32)reloc_idx; + } + radsort(sorted_relocs, relocs.count, lnk_reloc_sort_key_is_before); + for EachIndex(reloc_idx, relocs.count) { + COFF_Reloc *reloc = &sorted_relocs[reloc_idx].reloc; // error check relocation if (obj->header.machine == COFF_MachineType_X64) { @@ -4572,8 +5245,10 @@ THREAD_POOL_TASK_FUNC(lnk_obj_reloc_patcher) // commit new reloc value MemoryCopy(section_data.str + reloc->apply_off, &reloc_result, reloc_value.size); } + temp_end(reloc_temp); } + scratch_end(scratch); ProfEnd(); } @@ -4993,6 +5668,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_virtual_offsets_and_sizes_in_obj_section_headers ProfBeginV("Patch Virtual Offset And Size In Section Headers [%S]", obj->path); COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(obj->data, obj->header.section_table_range).str; + // headers live in the read-only mapped input view; promote the whole table in one call + // instead of taking a copy-on-write fault per page (~20 pages/obj on section-heavy objs) + lnk_cow_promote_range(section_table, dim_1u64(obj->header.section_table_range)); for (U64 sect_idx = 0; sect_idx < obj->header.section_count_no_null; sect_idx += 1) { COFF_SectionHeader *sect_header = §ion_table[sect_idx]; if (~obj->section_flags[sect_idx] & COFF_SectionFlag_LnkRemove) { @@ -5014,6 +5692,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_file_offsets_and_sizes_in_obj_section_headers_ta ProfBeginV("Patch File Offsets And Sizes In Obj Section Headers [%S]", obj->path); COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(obj->data, obj->header.section_table_range).str; + // usually a no-op: the virtual-offset patch task already promoted (and dirtied) these pages + lnk_cow_promote_range(section_table, dim_1u64(obj->header.section_table_range)); for (U64 sect_idx = 0; sect_idx < obj->header.section_count_no_null; sect_idx += 1) { COFF_SectionHeader *sect_header = §ion_table[sect_idx]; COFF_SectionFlags sect_flags = obj->section_flags[sect_idx]; @@ -5663,10 +6343,18 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfScope("Gather Sections") { TP_Temp temp = tp_temp_begin(arena); + // BARRIER pass (path B): the task synchronizes with barrier_wait(tp->barrier), so under + // /RAD_SHARED_THREAD_POOL it must run at a pinned cohort via the reserve path -- a plain + // tp_for_parallel admits workers incrementally and the barrier never fills (deadlock). + // Pin the cohort BEFORE sizing the per-lane ranges/defns so everything agrees on the width. + U32 C = tp_barrier_begin(tp); task.u.gather_sects.arena = arena->v[0]; - task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, tp->worker_count); - task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, tp->worker_count); - tp_for_parallel_prof(tp, arena, tp->worker_count, lnk_gather_sections_task, &task, "Gather Sections"); + task.u.gather_sects.ranges = tp_divide_work(arena->v[0], objs_count, C); + task.u.gather_sects.defns = push_array(arena->v[0], HashTable *, C); + ProfBegin("Gather Sections"); + tp_for_parallel_reserve(tp, arena, C, lnk_gather_sections_task, &task); // BARRIER pass (path B) + ProfEnd(); + tp_barrier_end(tp); tp_temp_end(temp); } @@ -5830,6 +6518,29 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // flag debug symbols to prevent them from being patched in subsequent passes tp_for_parallel_prof(tp, 0, objs_count, lnk_flag_debug_symbols_task, &task, "Flag Debug Symbols"); + // The patch passes below store final section numbers / values into every regular + // symbol record. The symbol tables live in the copy-on-write input mapping, so + // patching them in place copy-on-writes one page per touched symbol-table page + // (~1.2M pages / ~4.5GB of CoW commit on a large editor link). Give each obj a + // private copy of its symbol table up front -- lnk_coff_symbol_table_from_obj + // prefers the copy, and every patcher writes through symbol.raw_symbol pointers + // derived from it, so the input mapping stays clean. One sequential memcpy per + // obj costs the same bytes the CoW faults would have copied anyway, without the + // per-page fault + zero + charge machinery. + ProfScope("Copy Symbol Tables") + { + U64 *symtab_offsets = push_array_no_zero(temp.arena, U64, objs_count + 1); + U64 total_size = 0; + for EachIndex(obj_idx, objs_count) { + symtab_offsets[obj_idx] = total_size; + total_size += dim_1u64(objs[obj_idx]->header.symbol_table_range); + } + symtab_offsets[objs_count] = total_size; + task.u.patch_symtabs.symtab_copy_base = push_array_no_zero(lnk_get_huge_arena(), U8, total_size); + task.u.patch_symtabs.symtab_copy_offsets = symtab_offsets; + tp_for_parallel_prof(tp, 0, objs_count, lnk_copy_symbol_tables_task, &task, "Copy Symbol Tables"); + } + // patch symbols tp_for_parallel_prof(tp, 0, objs_count, lnk_patch_comdat_leaders_task, &task, "COMDAT Leaders" ); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_patch_common_block_leaders_task, &task, "Common Block Leaders"); @@ -5902,16 +6613,56 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); - for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { - LNK_Section *sect = §_n->data; - ProfBeginV("Section: %S Size: %M", sect->name, sect->fsize); - U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; - MemorySet(image_data.str + sect->foff, fill_byte, sect->fsize); - ProfEnd(); + { + // This is the first touch of the freshly committed ~image-size buffer: every + // page is a demand-zero fault. Serial on main, that soft-fault storm (plus the + // stream-set itself) parks all workers for the duration; range-split it across + // the pool instead. Split points are PAGE-ALIGNED in the image buffer (the + // reservation is page-aligned) so no two workers ever touch the same 4K page. + // Writes are value-identical to the serial loop and byte-disjoint -> byte-safe. + Temp fill_temp = temp_begin(scratch.arena); + U64 range_quantum = MB(4); + + // upper bound on range count + U64 range_cap = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + range_cap += CeilIntegerDiv(sect_n->data.fsize, range_quantum) + 1; + } + + LNK_FillAlignRange *ranges = push_array_no_zero(fill_temp.arena, LNK_FillAlignRange, range_cap); + U64 range_count = 0; + for EachNode(sect_n, LNK_SectionNode, sectab->list.first) { + LNK_Section *sect = §_n->data; + U8 fill_byte = sect->flags & COFF_SectionFlag_CntCode ? coff_code_align_byte_from_machine(config->machine) : 0; + U64 pos = sect->foff; + U64 end = sect->foff + sect->fsize; + for (; pos < end; ) { + U64 next = AlignDownPow2(pos + range_quantum, KB(4)); + next = ClampTop(next, end); + if (next <= pos) { next = end; } + Assert(range_count < range_cap); + LNK_FillAlignRange *range = &ranges[range_count++]; + range->dst = image_data.str + pos; + range->size = next - pos; + range->byte = fill_byte; + pos = next; + } + } + + // write-once into the image buffer -> stream past the cache (see lnk_stream_set); + // each task sfences its own NT stores before signalling completion, so after the + // join every fill below is globally visible to the contrib-fill / reloc passes + tp_for_parallel(tp, 0, range_count, lnk_fill_align_bytes_task, ranges); + + temp_end(fill_temp); } ProfEnd(); @@ -5977,7 +6728,7 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT // patch relocs { LNK_ObjRelocPatcher task = { .image_data = image_data, .objs = objs, .image_base = pe.image_base, .image_section_table = image_section_table }; - tp_for_parallel_prof(tp, 0, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); + tp_for_parallel_prof(tp, arena, objs_count, lnk_obj_reloc_patcher, &task, "Patch Relocs"); // arena: debug sections are patched-on-copy into private memory } // patch load config @@ -6417,11 +7168,270 @@ internal void lnk_write_thread(void *raw_ctx) { ProfBeginFunction(); + lnk_summary_phase_begin(LNK_SummaryPhase_Write); LNK_WriteThreadContext *ctx = raw_ctx; lnk_write_data_to_file_path(ctx->path, ctx->temp_path, ctx->data); + lnk_summary_phase_end(LNK_SummaryPhase_Write); ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ One-line end-of-link summary (always on; production triage). Everything +// needed at print time is stashed in this global as the link progresses, so +// the line can be emitted best-effort from ANY exit path (lnk_exit on error, +// entry_point on success) with whatever was known by then. + +typedef struct LNK_SummaryInfo +{ + volatile U32 printed; // print-exactly-once latch + U64 start_us; // set first thing in entry_point + U64 t0_ms; // UTC ms epoch at link start (t1 is stamped at print time) + U64 worker_count; + U64 objs_count; + U64 input_bytes; // sum of obj data sizes (lib members count their slice) + U64 libs_count; + // physical-memory samples (GlobalMemoryStatusEx): storm triage -- prod storms + // show pdb-phase kernel time exploding 54x for the same fault count, fitting + // free-list exhaustion / page-repurpose; these 3 samples prove/refute that + U64 mem_avail_t0; // ullAvailPhys at link start + U64 mem_avail_pdb; // ullAvailPhys at pdb-phase start (0 = phase never ran) + U32 mem_load_max; // max dwMemoryLoad seen across the samples + // name COPIES: config strings parsed out of an @rsp point into the response + // file buffer, whose scratch dies right after config parse -- capture the + // bytes here instead of keeping String8s into freed memory + U64 out_name_size; + U64 pool_name_size; // non-zero => /RAD_SHARED_THREAD_POOL + U8 out_name [128]; + U8 pool_name[128]; +} LNK_SummaryInfo; + +global LNK_SummaryInfo g_summary_info; + +internal void +lnk_summary_copy_name(U8 *dst, U64 dst_cap, U64 *dst_size_out, String8 name) +{ + U64 size = Min(name.size, dst_cap); + MemoryCopy(dst, name.str, size); + *dst_size_out = size; +} + +internal U64 +lnk_summary_us_from_timer(LNK_TimerType timer) +{ + // guard against a fatal exit mid-phase (begin stamped, end still zero) + return g_timers[timer].end > g_timers[timer].begin ? g_timers[timer].end - g_timers[timer].begin : 0; +} + +internal LNK_SummaryCounters +lnk_summary_counters_from_timer(LNK_TimerType timer) +{ + LNK_SummaryCounters zero = {0}; + // same mid-phase guard as lnk_summary_us_from_timer + if (g_timers[timer].end <= g_timers[timer].begin) { return zero; } + return lnk_summary_counters_sub_sat(g_timer_counters_end[timer], g_timer_counters_begin[timer]); +} + +// one GlobalMemoryStatusEx sample for the summary line: returns available +// physical bytes and folds dwMemoryLoad into the running max. 1 syscall per +// call, called 3x per link (link start, pdb-phase start, print time). +internal U64 +lnk_summary_sample_mem(void) +{ + U64 avail = 0; +#if OS_WINDOWS + MEMORYSTATUSEX msx = { sizeof(msx) }; + if (GlobalMemoryStatusEx(&msx)) { + avail = msx.ullAvailPhys; + if (msx.dwMemoryLoad > g_summary_info.mem_load_max) { g_summary_info.mem_load_max = msx.dwMemoryLoad; } + } +#endif + return avail; +} + +internal U64 +lnk_summary_utc_ms(void) +{ +#if OS_WINDOWS + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + U64 t100 = ((U64)ft.dwHighDateTime << 32) | ft.dwLowDateTime; + return (t100 - 116444736000000000ULL) / 10000; // FILETIME epoch -> unix ms epoch +#else + return 0; +#endif +} + +// one phase bucket -> "wall-ms/user-ms/kernel-ms/faults-K" (process-wide deltas +// at the bucket's boundaries; user can exceed wall on parallel phases, and a +// bucket that overlaps another thread's work counts that work too) +internal String8 +lnk_summary_str_from_counters(Arena *arena, LNK_SummaryCounters c) +{ + return push_str8f(arena, "%llu/%llu/%llu/%llu", c.wall_us / 1000, c.user_us / 1000, c.kern_us / 1000, c.faults / 1000); +} + +internal void +lnk_print_summary(int exit_code) +{ + // run exactly once, no matter which exit path gets here first + if (ins_atomic_u32_eval_cond_assign(&g_summary_info.printed, 1, 0) != 0) { + return; + } + + // detach from the shared-pool cross-process counter on every exit path, even + // when the summary line is off -- the linker leaves through _exit and never + // runs tp_release, so this is the only place the counter gets decremented + F64 pool_grant_avg = 0, pool_park_seconds = 0; + U32 pool_procs_now = 0, pool_procs_peak = 0; + B32 pool_on = (g_summary_info.pool_name_size > 0); + if (pool_on) { + tp_stats_snapshot(&pool_grant_avg, &pool_park_seconds); + tp_procs_snapshot(&pool_procs_now, &pool_procs_peak); + tp_procs_detach(); + } + + // the line itself is opt-in (/RAD_LOG:Summary) -- always-on turned out to be + // noise in build logs; farm convoy triage passes the switch explicitly + if (!lnk_get_log_status(LNK_Log_Summary)) { + return; + } + + Temp scratch = scratch_begin(0, 0); + + F64 wall = g_summary_info.start_us ? (F64)(now_time_us() - g_summary_info.start_us) / 1000000.0 : 0; + + // process CPU + memory counters + F64 user_time = 0, kernel_time = 0, peak_ws_gib = 0, page_faults_m = 0, peak_commit_gib = 0; + U32 cow_promoted_pages = 0; +#if OS_WINDOWS + { + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + user_time = (F64)(((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10000000.0; + kernel_time = (F64)(((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10000000.0; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + peak_ws_gib = (F64)pmc.PeakWorkingSetSize / (F64)GB(1); + page_faults_m = (F64)pmc.PageFaultCount / 1000000.0; + // peak pagefile-backed commit charge -- the number build-farm memory admission + // sees; with read-only input views this tracks ws minus the mapped input set + peak_commit_gib = (F64)pmc.PeakPagefileUsage / (F64)GB(1); + } + cow_promoted_pages = (U32)g_lnk_cow_promoted_pages; + } +#endif + + // process IO totals: hard page-ins on mapped inputs surface as read bytes, + // UBA-detoured output writes as write bytes + U64 io_read_mb = 0, io_write_mb = 0; +#if OS_WINDOWS + { + IO_COUNTERS ioc = {0}; + if (GetProcessIoCounters(GetCurrentProcess(), &ioc)) { + io_read_mb = ioc.ReadTransferCount / MB(1); + io_write_mb = ioc.WriteTransferCount / MB(1); + } + } +#endif + + // phase triplets. img/dbg/pdb come from the /RAD_LOG:TIMERS stamps; dbg is + // the debug-info umbrella minus the PDB/RDI sub-phases it contains. + LNK_SummaryCounters img_c = lnk_summary_counters_from_timer(LNK_Timer_Image); + LNK_SummaryCounters pdb_c = lnk_summary_counters_from_timer(LNK_Timer_Pdb); + LNK_SummaryCounters rdi_c = lnk_summary_counters_from_timer(LNK_Timer_Rdi); + LNK_SummaryCounters dbg_c = lnk_summary_counters_sub_sat(lnk_summary_counters_sub_sat(lnk_summary_counters_from_timer(LNK_Timer_Debug), pdb_c), rdi_c); + + // residual catch-alls: umbrella bucket minus the sum of its printed + // sub-buckets, clamped at 0 per field (a /PDBSTRIPPED link runs the pdb + // sub-phases a second time OUTSIDE the Timer_Pdb bracket, which can push the + // sub-bucket sum past the umbrella -- clamp instead of printing garbage). + // Storm triage: prod shows the pdbg sub-buckets covering only ~19% of pdb + // kernel time in a storm window vs ~96% locally -- other= pins the + // uncovered span without waiting for a local repro. + LNK_SummaryCounters pdb_other, dbg_other; + { + LNK_SummaryCounters pdbg_sum = g_summary_phase[LNK_SummaryPhase_PdbGsi]; + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbHsh]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbIni]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSym]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMod]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbTpi]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbStr]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbSc]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbMsf]); + pdbg_sum = lnk_summary_counters_add(pdbg_sum, g_summary_phase[LNK_SummaryPhase_PdbWr]); + pdb_other = lnk_summary_counters_sub_sat(pdb_c, pdbg_sum); + + LNK_SummaryCounters dbgg_sum = lnk_summary_counters_add(g_summary_phase[LNK_SummaryPhase_DbgMcvi], g_summary_phase[LNK_SummaryPhase_DbgMerge]); + dbg_other = lnk_summary_counters_sub_sat(dbg_c, dbgg_sum); + } + + // governor stats snapshotted above, before the detach + String8 pool_stats = str8_zero(); + if (pool_on) { + pool_stats = push_str8f(scratch.arena, " pool=%S grant_avg=%.1f park=%.1f procs=%u/%u", + str8(g_summary_info.pool_name, g_summary_info.pool_name_size), pool_grant_avg, pool_park_seconds, pool_procs_now, pool_procs_peak); + } + + // final memory sample (t1) -- 3rd and last GlobalMemoryStatusEx of the link + U64 mem_avail_t1 = lnk_summary_sample_mem(); + + lnk_fprintf(stdout, + "[radlink summary] v=3 out=%S exit=%d t0=%llu t1=%llu wall=%.1f user=%.1f kern=%.1f ws=%.1fG cm=%.1fG cowp=%u pf=%.1fM io=%llu/%lluMB mem=%.1f/%.1f/%.1f/%u workers=%llu%S" + " in=%lluo/%.1fG libs=%llu" + " ph[inp=%S res=%S icf=%S ref=%S img=%S dbg=%S pdb=%S wr=%S]" + " dbgg[mcvi=%S merge=%S other=%S]" + " pdbg[hsh=%S ini=%S gsi=%S sym=%S mod=%S tpi=%S str=%S sc=%S msf=%S wr=%S other=%S]\n", + g_summary_info.out_name_size ? str8(g_summary_info.out_name, g_summary_info.out_name_size) : str8_lit("-"), + exit_code, + g_summary_info.t0_ms, + lnk_summary_utc_ms(), + wall, + user_time, + kernel_time, + peak_ws_gib, + peak_commit_gib, + cow_promoted_pages, + page_faults_m, + io_read_mb, + io_write_mb, + (F64)g_summary_info.mem_avail_t0 / (F64)GB(1), + (F64)g_summary_info.mem_avail_pdb / (F64)GB(1), + (F64)mem_avail_t1 / (F64)GB(1), + g_summary_info.mem_load_max, + g_summary_info.worker_count, + pool_stats, + g_summary_info.objs_count, + (F64)g_summary_info.input_bytes / (F64)GB(1), + g_summary_info.libs_count, + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Input]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Resolve]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Icf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Ref]), + lnk_summary_str_from_counters(scratch.arena, img_c), + lnk_summary_str_from_counters(scratch.arena, dbg_c), + lnk_summary_str_from_counters(scratch.arena, pdb_c), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_Write]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMcvi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_DbgMerge]), + lnk_summary_str_from_counters(scratch.arena, dbg_other), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbHsh]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbIni]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbGsi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSym]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMod]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbTpi]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbStr]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbSc]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbMsf]), + lnk_summary_str_from_counters(scratch.arena, g_summary_phase[LNK_SummaryPhase_PdbWr]), + lnk_summary_str_from_counters(scratch.arena, pdb_other)); + + scratch_end(scratch); +} + + internal void lnk_log_timers(void) { @@ -6451,10 +7461,105 @@ lnk_log_timers(void) StringJoin new_line_join = { str8_lit_comp(""), str8_lit_comp("\n"), str8_lit_comp("") }; String8 output = str8_list_join(scratch.arena, &output_list, &new_line_join); lnk_log(LNK_Log_Timers, "%S\n", output); - + + // Diagnostic: when RADLINK_PHASE_LOG is set, also write machine-parseable raw + // per-phase micros to that file (for automated perf A/B). Env-unset -> no-op, + // so normal/validation links are byte-identical; this never touches DLL/PDB bytes. + char *phase_log_path = getenv("RADLINK_PHASE_LOG"); + if (phase_log_path != 0 && phase_log_path[0] != 0) { + String8List raw_list = {0}; + for (U64 i = 0; i < LNK_Timer_Count; ++i) { + str8_list_pushf(scratch.arena, &raw_list, "%S %llu\n", lnk_string_from_timer_type(i), g_timers[i].end - g_timers[i].begin); + } + str8_list_pushf(scratch.arena, &raw_list, "TOTAL %llu\n", total_build_time_micro); + String8 raw_str = str8_list_join(scratch.arena, &raw_list, 0); + lnk_write_data_to_file_path(str8_cstring(phase_log_path), str8_zero(), raw_str); + } + scratch_end(scratch); } +// scratch free-list blocks detached during the decommit pass, released on a +// background thread (see lnk_scratch_decommit_worker) +global Arena *g_detached_scratch_blocks = 0; +global Thread g_scratch_freelist_reaper = {0}; + +internal +THREAD_POOL_TASK_FUNC(lnk_scratch_decommit_worker) +{ + // Each worker decommits the committed-but-unused pages of its OWN equipped + // tctx scratch arenas. Runs on the worker thread, so tctx_selected() yields + // that worker's scratch. No cross-thread arena access. + // + // The barrier (dispatched with task_count == worker_count) guarantees every + // worker runs the body exactly once -- otherwise the work-stealing loop could + // let one fast worker grab several tasks and leave other workers' scratch + // committed. All worker_count threads are woken, so all must reach the barrier. + // + // NOTE(perf): redistributing these decommits in chunks across the pool does + // NOT help: MEM_DECOMMIT serializes in the kernel on the process address-space + // lock (~14 GB/s aggregate no matter the thread count; measured 9.3 GiB in + // 677 ms chunked-parallel vs ~500 ms with this per-worker scheme). And handing + // the ACTIVE-CHAIN decommit to a background thread is UNSAFE here: workers push + // to these scratch arenas as soon as the PDB build starts, and a push would + // re-commit pages that the background decommit then rips out. + // + // The FREE-LIST blocks are different: they hold no live data and are only + // touched again when a grow pops them. On the editor link ~84% of the + // decommitted bytes (9.4 of 11.3 GiB) sit in free-list blocks, so instead of + // decommitting them here (serialized kernel work on the critical path), each + // worker DETACHES its arenas' free chains (pointer ops, same thread => safe) + // onto a global list that a background thread releases while the PDB build + // runs. A post-detach grow simply sees an empty free list and reserves a + // fresh block -- same cost as the re-commit it would have paid anyway. + TCTX *tctx = tctx_selected(); + for EachIndex(arena_idx, ArrayCount(tctx->arenas)) { + Arena *arena = tctx->arenas[arena_idx]; + if (arena == 0) { continue; } +#if ARENA_FREE_LIST + // detach this arena's free chain and publish the blocks for background release + for (Arena *block = arena->free_last, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + for (;;) { + Arena *head = (Arena *)ins_atomic_u64_eval(&g_detached_scratch_blocks); + block->prev = head; + if ((Arena *)ins_atomic_u64_eval_cond_assign((U64 *)&g_detached_scratch_blocks, (U64)block, (U64)head) == head) { break; } + } + } + arena->free_last = 0; +#endif + // decommit the committed-but-unused pages above the live pos (active chain) + arena_decommit_unused(arena); + } + barrier_wait(tp->barrier); +} + +// Releases the scratch free-list blocks detached by lnk_scratch_decommit_worker. +// Runs in the background: MEM_RELEASE serializes on the process address-space +// lock in the kernel, so on the main thread this would extend the decommit +// window 1:1; off the main thread it overlaps the PDB build. +internal void +lnk_detached_scratch_release_thread(void *raw) +{ + ProfBeginFunction(); + U64 begin_us = now_time_us(); + + U64 released_bytes = 0; + U64 released_count = 0; + Arena *chain = (Arena *)ins_atomic_u64_eval_assign((U64 *)&g_detached_scratch_blocks, 0); + for (Arena *block = chain, *block_next = 0; block != 0; block = block_next) { + block_next = block->prev; + released_bytes += block->cmt; + released_count += 1; + AsanUnpoisonMemoryRegion(block, block->cmt); + release_memory(block, block->res); + } + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu detached scratch blocks (%llu MiB committed) took %.2f ms (off main thread)", + released_count, released_bytes / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_p2r_worker) { @@ -6510,6 +7615,52 @@ lnk_debug_filter_objs(Arena *arena, LNK_Obj **objs, U64 objs_count, U64 *count_o return debug_info_objs; } +// Parallel release of memory-mapped input file views. +// Inputs are mapped copy-on-write (PAGE_WRITECOPY/FILE_MAP_COPY); pages touched +// during linking become private-dirty and are reclaimed by the kernel in +// single-threaded process rundown at exit (~3s for a large link). Unmapping them +// in parallel before exit moves that reclaim off the serial post-exit path. +typedef struct LNK_UnmapViewTask +{ + String8 *views; +} LNK_UnmapViewTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_unmap_view_task) +{ + LNK_UnmapViewTask *task = raw_task; + String8 view = task->views[task_id]; +#if OS_WINDOWS + UnmapViewOfFile(view.str); +#elif OS_LINUX + munmap(view.str, view.size); +#endif +} + +internal void +lnk_release_input_views(TP_Context *tp, LNK_Inputer *inputer) +{ + Temp scratch = scratch_begin(0, 0); + + // collect distinct whole-file mapped views (is_thin); skip lib-member + // substrings and linkgen arena data + U64 cap = inputer->objs.count + inputer->libs.count; + String8 *views = push_array_no_zero(scratch.arena, String8, cap); + U64 count = 0; + for EachNode(n, LNK_Input, inputer->objs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + for EachNode(n, LNK_Input, inputer->libs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + + if (count > 0) { + U64 begin_us = now_time_us(); + LNK_UnmapViewTask task = { .views = views }; + tp_for_parallel(tp, 0, count, lnk_unmap_view_task, &task); + U64 end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "Released %llu input views in %.2f ms", count, (F64)(end_us - begin_us) / 1000.0); + } + + scratch_end(scratch); +} + internal void lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) { @@ -6537,6 +7688,15 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_Obj **objs = lnk_array_from_obj_list(scratch.arena, link.objs); LNK_Lib **libs = lnk_array_from_lib_list(scratch.arena, link.libs); + // summary: input volume (lib members count their member slice) + { + U64 input_bytes = 0; + for EachIndex(obj_idx, objs_count) { input_bytes += objs[obj_idx]->data.size; } + g_summary_info.objs_count = objs_count; + g_summary_info.libs_count = libs_count; + g_summary_info.input_bytes = input_bytes; + } + // // Layout Image // @@ -6589,8 +7749,38 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // CodeView // LNK_RRT_Array rrt_input = lnk_rrt_array_from_config(arena->v[0], config); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMcvi); LNK_CodeViewInput cv = lnk_make_code_view_input(tp, arena, config, debug_info_objs_count, debug_info_objs, rrt_input); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMcvi); + lnk_summary_phase_begin(LNK_SummaryPhase_DbgMerge); LNK_MergedTypes cv_types = lnk_merge_types(tp, arena, &cv, 0); + lnk_summary_phase_end(LNK_SummaryPhase_DbgMerge); + + // prune merged types not reachable from any surviving symbol (PDB-size win). OFF by default: + // it removes types that a debugger can still legitimately cast to in the watch window + // (reachable-from-symbols is a subset of castable-types). Opt in with /OPT:GCTYPES. + if (config->opt_gc_types == LNK_SwitchState_Yes) { + lnk_gc_types(tp, arena->v[0], &cv, &cv_types); + } + + // merge-types reached the scratch high-water (~9GB of per-thread tctx scratch + // stays committed but idle). Release those unused scratch pages back to the OS + // before the PDB build re-grows, dropping the recorded peak working set. Each + // worker decommits its own scratch; do the main thread's scratch too. Only + // pages strictly above each arena's live `pos` are touched, so output stays + // byte-identical and the push path re-commits on demand during PDB build. + { + ProfBegin("Decommit Scratch"); + U64 decommit_begin_us = now_time_us(); + // task_count == worker_count + the in-worker barrier => every worker + // (worker 0 IS the main thread) runs exactly once, covering main's scratch. + tp_for_parallel_reserve(tp, 0, tp->worker_count, lnk_scratch_decommit_worker, 0); // BARRIER pass (path B) + if (g_detached_scratch_blocks != 0) { + g_scratch_freelist_reaper = thread_launch(lnk_detached_scratch_release_thread, 0); + } + lnk_log(LNK_Log_Timers, "[teardown] scratch decommit pass in %.2f ms", (F64)(now_time_us() - decommit_begin_us) / 1000.0); + ProfEnd(); + } // // Debug Info @@ -6600,8 +7790,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) if (config->debug_mode == LNK_DebugMode_Full || config->rad_debug == LNK_SwitchState_Yes) { LNK_FileArtifact pdb_artifact = {0}; { + g_summary_info.mem_avail_pdb = lnk_summary_sample_mem(); lnk_timer_begin(LNK_Timer_Pdb); - + lnk_summary_phase_begin(LNK_SummaryPhase_PdbHsh); if (config->pdb_hash_type_names != LNK_TypeNameHashMode_None) { lnk_replace_type_names_with_hashes(tp, arena, @@ -6611,10 +7802,12 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) config->pdb_hash_type_name_length, config->pdb_hash_type_name_map); } - + lnk_summary_phase_end(LNK_SummaryPhase_PdbHsh); pdb_writer.output_path = config->debug_mode == LNK_DebugMode_Full ? config->pdb_name : str8_zero(); pdb_writer.temp_output_path = config->debug_mode == LNK_DebugMode_Full ? config->temp_pdb_name : str8_zero(); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &cv, cv_types, pdb_writer, LNK_PDB_BuilderFlag_All); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); lnk_timer_end(LNK_Timer_Pdb); } @@ -6623,7 +7816,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) lnk_timer_begin(LNK_Timer_Rdi); LNK_P2R p2r = { .config = config, .pdb_data = lnk_data_from_file_artifact(lnk_get_huge_arena(), &pdb_artifact), .image_data = image_ctx.image_data }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_p2r_worker, &p2r); // BARRIER pass (path B) String8List rdi_blobs = rdim_file_blobs_from_section_bundle(scratch.arena, &p2r.bake_results.section_bundle); lnk_write_data_list_to_file_path(config->rad_debug_name, config->temp_rad_debug_name, rdi_blobs); @@ -6675,9 +7868,11 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) TryReadBreak(cv_read_symbol(n->string, cursor, CV_SymbolAlign, &symbol), cursor); if (symbol.kind == CV_SymKind_SKIP) { continue; } if (cv_is_lproc(symbol)) { - CV_SymProc32 *src_proc = str8_deserial_get_raw_ptr(symbol.data, 0, sizeof(*src_proc)); - memory_write32(&src_proc->itype, 0); // strip type index + // strip the type index in the DESTINATION copy -- the source $$S stays untouched + // (patching the source would dirty its private/CoW backing pages for no reason) + U64 rec_off = buffer_cursor; buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &symbol, CV_SymbolAlign); + memory_write32(buffer + rec_off + sizeof(CV_SymbolHeader) + OffsetOf(CV_SymProc32, itype), 0); buffer_cursor += cv_write_symbol(buffer, buffer_cursor, buffer_size, &(CV_Symbol){ .kind = CV_SymKind_END }, CV_SymbolAlign); } } @@ -6698,7 +7893,9 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) stripped_cv.symbol_input_ranges = push_array(scratch.arena, Rng1U64, tp->worker_count); LNK_FileArtifact pdb_artifact = lnk_build_pdb(tp, arena, image_ctx.image_data, config, symtab, &stripped_cv, (LNK_MergedTypes){0}, (LNK_PdbWriter){0}, LNK_PDB_BuilderFlag_All); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbWr); lnk_write_data_list_to_file_path(config->pdb_stripped_name, str8f(scratch.arena, "%S.tmp", config->pdb_stripped_name), pdb_artifact.data); + lnk_summary_phase_end(LNK_SummaryPhase_PdbWr); } lnk_timer_end(LNK_Timer_Debug); @@ -6719,11 +7916,46 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // reap the background arena-release thread, if one is still in flight + if (g_arena_reaper_thread.u64[0] != 0) { + thread_join(g_arena_reaper_thread, max_U64); + MemoryZeroStruct(&g_arena_reaper_thread); + } + + // reap the background scratch free-list release thread, if one was launched + if (g_scratch_freelist_reaper.u64[0] != 0) { + thread_join(g_scratch_freelist_reaper, max_U64); + MemoryZeroStruct(&g_scratch_freelist_reaper); + } + + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); + + // outputs are written and inputs are no longer read; release the copy-on-write + // input views in parallel so their dirty pages are reclaimed here (multi-threaded) + // instead of in single-threaded process rundown at exit. Only safe for the CoW + // (read-only) mapping mode; read-write-shared would flush dirty pages back to the + // input files on unmap. + // + // Only worth it when the process stays alive after the link (shared thread pool + // serving multiple links). When we exit right after this, the views are mostly + // clean now (debug relocs/TI fixups are patched-on-copy) and bulk process-exit + // teardown reclaims them cheaper than an explicit unmap pass. + if (lnk_is_thread_pool_shared(config) && + (config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + lnk_release_input_views(tp, inputer); + } + // // Timers // - if (lnk_get_log_status(LNK_Log_Timers)) { - lnk_log_timers(); + { + char *phase_log_env = getenv("RADLINK_PHASE_LOG"); + if (lnk_get_log_status(LNK_Log_Timers) || (phase_log_env != 0 && phase_log_env[0] != 0)) { + lnk_log_timers(); + } } scratch_end(scratch); @@ -6897,7 +8129,7 @@ lnk_run_type_server(TP_Context *tp, TP_Arena *arena, LNK_Config *config) ProfScope("Pack Type Data & Data Ranges") { LNK_RRTTypeDataSerializer task = { &cv_types, &rrt.type_data_raw, rrt.type_data_ranges }; - tp_for_parallel(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); + tp_for_parallel_reserve(tp, arena, tp->worker_count, lnk_serialize_rrt_type_data_task, &task); // BARRIER pass (path B) // pack type index ranges for EachIndex(i, CV_TypeIndexSource_COUNT) { @@ -6983,9 +8215,21 @@ internal void entry_point(CmdLine *cmdline) { Temp scratch = scratch_begin(0,0); + g_summary_info.start_us = now_time_us(); + g_summary_info.t0_ms = lnk_summary_utc_ms(); + g_summary_info.mem_avail_t0 = lnk_summary_sample_mem(); lnk_log_begin(); - LNK_Config *config = lnk_config_from_argcv(cmdline); + LNK_Config *config = lnk_config_from_argcv(cmdline); + + // summary: identity fields, copied IMMEDIATELY after the parse -- @rsp-parsed + // config strings can point into the response-file scratch, which is reused by + // the very next pushes on this thread (observed as garbage pool= names when + // the copy happened after tp_alloc) + lnk_summary_copy_name(g_summary_info.out_name, sizeof(g_summary_info.out_name), &g_summary_info.out_name_size, str8_skip_last_slash(config->out_path)); + lnk_summary_copy_name(g_summary_info.pool_name, sizeof(g_summary_info.pool_name), &g_summary_info.pool_name_size, config->shared_thread_pool_name); + g_summary_info.worker_count = config->worker_count; + TP_Context *tp = tp_alloc(scratch.arena, config->worker_count, config->max_worker_count, config->shared_thread_pool_name); TP_Arena *tp_arena = tp_arena_alloc(tp); @@ -7017,6 +8261,8 @@ entry_point(CmdLine *cmdline) case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; } + lnk_print_summary(0); + lnk_log_end(); scratch_end(scratch); } diff --git a/src/linker/lnk.h b/src/linker/lnk.h index 7d27c2470..0edbce112 100644 --- a/src/linker/lnk.h +++ b/src/linker/lnk.h @@ -295,6 +295,8 @@ typedef struct Rng1U64 *common_block_ranges; LNK_CommonBlockContrib *common_block_contribs; COFF_SymbolValueInterpType fixup_type; + U8 *symtab_copy_base; // private symbol-table copies (patchers write here, not the CoW input mapping) + U64 *symtab_copy_offsets; // [objs_count+1] byte offsets into symtab_copy_base } patch_symtabs; struct { String8 image_data; @@ -400,7 +402,9 @@ internal LNK_LinkResult lnk_link_image (TP_Context *tp, TP_Arena *arena, LNK_Con // --- Optimizations ----------------------------------------------------------- internal void lnk_opt_ref(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); -internal void lnk_opt_icf(TP_Context *tp, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal void lnk_opt_icf(TP_Context *tp, Arena *perm, LNK_SymbolTable *symtab, LNK_Config *config, LNK_Obj **objs, U64 objs_count); +internal U32 lnk_icf_debug_s_child_from_section(LNK_Obj *obj, U32 fn_sn); +internal void lnk_icf_mark_folded_lines(TP_Context *tp, TP_Arena *arena, LNK_Obj **objs, U64 objs_count); // --- Win32 Image ------------------------------------------------------------- @@ -413,3 +417,9 @@ internal LNK_ImageContext lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_C internal void lnk_log_link_stats(LNK_ObjList obj_list, LNK_LibList *lib_index, LNK_SectionTable *sectab); internal void lnk_log_timers(void); + +// One-line end-of-link summary for production triage (always on). Prints +// exactly once; safe to call from any exit path (values best-effort on early +// error exits). Defined in lnk.c; called from lnk_exit and entry_point. +internal void lnk_print_summary(int exit_code); + diff --git a/src/linker/lnk_config.c b/src/linker/lnk_config.c index eb08ca3ba..2a7f8e969 100644 --- a/src/linker/lnk_config.c +++ b/src/linker/lnk_config.c @@ -84,6 +84,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_EnvLib, 0, "RAD_ENV_LIB", "[:NO]", "Collect libraries from %%LIB%% and %%LIBPATH%% varibles." }, { LNK_CmdSwitch_Rad_Exe, 0, "RAD_EXE", "[:NO]", "Set EXE bit in the image header." }, { LNK_CmdSwitch_Rad_Guid, 0, "RAD_GUID", ":{IMAGEBLAKE3|XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXXXXXX}", "The image guid that is embeded in the debug info." }, + { LNK_CmdSwitch_Rad_IcfHashAlg, 0, "RAD_ICF_HASH_ALG", ":{BLAKE3|XXH3}", "Sets hashing algorithm for /OPT:ICF refinement round keys. Default BLAKE3." }, { LNK_CmdSwitch_Rad_LargePages, 0, "RAD_LARGE_PAGES", "[:NO]", "Disabled by default on Windows." }, { LNK_CmdSwitch_Rad_LinkVer, 0, "RAD_LINK_VER", ":##,##", "Linker version." }, { LNK_CmdSwitch_Rad_Log, 0, "RAD_LOG", ":{ALL,INPUT_OBJ,INPUT_LIB,IO,LINK_STATS,TIMERS}", "Loggers." }, @@ -103,6 +104,7 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_Rad_WriteTempFiles, 0, "RAD_WRITE_TEMP_FILES", "[:NO]", "When speicifed linker writes image and debug info to temporary files and renames after link is done." }, { LNK_CmdSwitch_Rad_TimeStamp, 0, "RAD_TIME_STAMP", ":#", "Time stamp embeded in EXE and PDB." }, { LNK_CmdSwitch_Rad_DebugTypeHash, 0, "RAD_DEBUG_TYPE_HASH", ":{BLAKE3|XXHASH}", "Sets hashing algorithm for debug type merging." }, + { LNK_CmdSwitch_Rad_DebugTypeHash, 0, "RAD_TYPEHASHALG", ":{BLAKE3|XXHASH}", "Alias of RAD_DEBUG_TYPE_HASH (spelling used by UnrealBuildTool)." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolLimit, 0, "RAD_UNRESOLVED_SYMBOL_LIMIT", ":#", "Limits number of unresolved symbol errors linker reports." }, { LNK_CmdSwitch_Rad_UnresolvedSymbolRefLimit, 0, "RAD_UNRESOLVED_SYMBOL_REF_LIMIT", ":#", "Limit number of unresolved symbol references linker reports." }, { LNK_CmdSwitch_Rad_Version, 0, "RAD_VERSION", "", "Print version and exit." }, @@ -112,6 +114,8 @@ global read_only LNK_CmdSwitch g_cmd_switch_map[] = { LNK_CmdSwitch_RadTypeServer, 0, "RAD_TYPE_SERVER", ":FILENAME", "Merge types and store them in the specified file. The filename must have the .rrt extension." }, { LNK_CmdSwitch_LLVM_AddrSig, 0, "LLVM_ADDRSIG", "[:NO]", "Use .llvm_addrsig to guide ICF." }, + { LNK_CmdSwitch_IfcMap, 1, "IFCMAP", ":FILENAME", "Map a header-unit module interface (.ifc) for debug-record resolution (TOML)." }, + { LNK_CmdSwitch_IfcDebugRecords, 0, "IFCDEBUGRECORDS", "[:NO]", "Resolve MSVC header-unit IFC debug records into real CodeView types." }, { LNK_CmdSwitch_Help, 0, "HELP", "", "" }, { LNK_CmdSwitch_Help, 0, "?", "", "" }, @@ -1256,7 +1260,18 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List switch (cmd_switch) { case LNK_CmdSwitch_Null: { String8 value = str8_list_join(scratch.arena, &value_strings, &(StringJoin){.sep=str8_lit_comp(",")}); - lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + + // Unknown /RAD_* switches on the command line warn and are ignored: the + // RAD_ namespace is owned by this linker, but newer build scripts must + // keep working against older radlink binaries (forward compatibility), + // so an unrecognized /RAD_* switch must not fail the link. Use + // LNK_Warning_Cmdl so the warning stays visible even though the + // release-default /RAD_IGNORE mutes LNK_Warning_UnknownSwitch. + if (obj == 0 && str8_match_lit("RAD_", str8_prefix(cmd_name, 4), StringMatchFlag_CaseInsensitive)) { + lnk_error(LNK_Warning_Cmdl, "unknown switch \"/%S%s%S\"; this radlink build does not support it -- switch ignored", cmd_name, value.size ? ":" : "", value); + } else { + lnk_error_obj(LNK_Warning_UnknownSwitch, obj, "unknown switch: \"/%S%s%S\"", cmd_name, value.size ? ":" : "", value); + } } break; default: break; @@ -1784,6 +1799,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List config->opt_lbr = LNK_SwitchState_Yes; } else if (str8_match_lit("nolibr", param, StringMatchFlag_CaseInsensitive)) { config->opt_lbr = LNK_SwitchState_No; + } else if (str8_match_lit("gctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_Yes; + } else if (str8_match_lit("nogctypes", param, StringMatchFlag_CaseInsensitive)) { + config->opt_gc_types = LNK_SwitchState_No; } else { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown option \"%S\"", param); } @@ -2261,7 +2280,10 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List if (value_strings.node_count == 0) { config->shared_thread_pool_name = str8_lit(LNK_DEFAULT_THREAD_POOL_NAME); } else { - lnk_cmd_switch_parse_string(obj, cmd_switch, value_strings, &config->shared_thread_pool_name); + // NOTE: must copy into the config arena -- the parsed string points into + // response-file/cmdline scratch that is freed long before late consumers + // (pool init, summary) read it + lnk_cmd_switch_parse_string_copy(config->arena, obj, cmd_switch, value_strings, &config->shared_thread_pool_name); if (config->shared_thread_pool_name.size == 0) { lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "invalid empty string for thread pool name"); } @@ -2325,6 +2347,19 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List lnk_cmd_switch_parse_u32(obj, cmd_switch, value_strings, &config->time_stamp, 0); } break; + case LNK_CmdSwitch_Rad_IcfHashAlg: { + String8 alg = {0}; + if (lnk_cmd_switch_parse_string(obj, cmd_switch, value_strings, &alg)) { + if (str8_match(alg, str8_lit("BLAKE3"), StringMatchFlag_CaseInsensitive)) { + config->icf_hash_xxh3 = 0; + } else if (str8_match(alg, str8_lit("XXH3"), StringMatchFlag_CaseInsensitive)) { + config->icf_hash_xxh3 = 1; + } else { + lnk_error_cmd_switch(LNK_Error_Cmdl, obj, cmd_switch, "unknown hash alg: %S", alg); + } + } + } break; + case LNK_CmdSwitch_Rad_DebugTypeHash: { String8 alg = {0}; if (lnk_cmd_switch_parse_string(obj, cmd_switch, value_strings, &alg)) { @@ -2385,6 +2420,17 @@ lnk_apply_cmd_option_to_config(LNK_Config *config, String8 cmd_name, String8List case LNK_CmdSwitch_LLVM_AddrSig: { lnk_cmd_switch_parse_flag(obj, cmd_switch, value_strings, &config->llvm_addrsig); } break; + case LNK_CmdSwitch_IfcMap: { + // collect .toml paths (header-unit -> .ifc); parsed lazily during debug-info build + String8List copy = str8_list_copy(config->arena, &value_strings); + str8_list_concat_in_place(&config->ifc_map_list, ©); + } break; + case LNK_CmdSwitch_IfcDebugRecords: { + LNK_SwitchState state = LNK_SwitchState_Null; + if (lnk_cmd_switch_parse_flag(obj, cmd_switch, value_strings, &state)) { + config->ifc_debug_records = state; + } + } break; } scratch_end(scratch); diff --git a/src/linker/lnk_config.h b/src/linker/lnk_config.h index 9fd5057f9..a26436514 100644 --- a/src/linker/lnk_config.h +++ b/src/linker/lnk_config.h @@ -115,6 +115,7 @@ typedef enum LNK_CmdSwitch_Rad_Exe, LNK_CmdSwitch_Rad_Guid, LNK_CmdSwitch_Rad_Ignore, + LNK_CmdSwitch_Rad_IcfHashAlg, LNK_CmdSwitch_Rad_ImageAltPath, LNK_CmdSwitch_Rad_LargePages, LNK_CmdSwitch_Rad_LinkVer, @@ -148,6 +149,8 @@ typedef enum LNK_CmdSwitch_RadTypeServer_MatchObj, LNK_CmdSwitch_LLVM_AddrSig, + LNK_CmdSwitch_IfcMap, + LNK_CmdSwitch_IfcDebugRecords, LNK_CmdSwitch_Help, @@ -314,7 +317,9 @@ typedef struct LNK_Config B32 ghash; LNK_SwitchState opt_ref; LNK_SwitchState opt_icf; + B32 icf_hash_xxh3; // /RAD_ICF_HASH_ALG:XXH3 -- XXH3-128 refinement round keys instead of blake3 LNK_SwitchState opt_lbr; + LNK_SwitchState opt_gc_types; // /OPT:GCTYPES -- prune unreferenced CodeView types. Default OFF: shrinks PDB but a pruned type can't be cast-to in the debugger watch window. U64 opt_iter_count; LNK_SwitchState import_table_emit_biat; LNK_SwitchState import_table_emit_uiat; @@ -415,6 +420,8 @@ typedef struct LNK_Config LNK_SwitchState type_server; LNK_SwitchState sort_imports; LNK_SwitchState llvm_addrsig; + LNK_SwitchState ifc_debug_records; // resolve LF_IFC_RECORD (0x1522) into real CodeView types + String8List ifc_map_list; // .toml paths from /ifcMap (header-unit -> .ifc) } LNK_Config; // --- MSVC Error Codes -------------------------------------------------------- diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index e217d057b..19d1ea7d4 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -7,11 +7,162 @@ internal Arena * lnk_get_huge_arena(void) { if (g_huge_arena == 0) { - g_huge_arena = arena_alloc(.name = "HUGE"); + // 2MB commit quantum (vs the 64KB default): this arena backs multi-GB debug + // info merges; the larger quantum cuts VirtualAlloc(MEM_COMMIT) syscalls + // (all serialized on the process address-space lock) ~32x for at most 2MB + // of slack past the high-water mark. + g_huge_arena = arena_alloc(.commit_size = MB(2), .name = "HUGE"); } return g_huge_arena; } +// Handle of the in-flight background arena-release thread (at most one). Joined +// (a) before launching the next reaper and (b) at the end of the link, next to +// the image-write-thread join. NOTE: do NOT thread_detach right after +// thread_launch -- that releases the W32_Entity the new thread's entry point is +// about to read (startup race). +static Thread g_arena_reaper_thread = {0}; + +internal void +lnk_arena_release_thread(void *raw_arena) +{ + // REAPER: releasing a huge arena costs ~50-100ms/GB of committed pages in the + // kernel (MiDeleteVaDirect/MiDecommitFreePage walk every PTE under + // VirtualFree(MEM_RELEASE)), and that work serializes on the process + // address-space lock -- chunking it across the thread pool does NOT make it + // faster (measured: 8 GiB in 860 ms chunked-parallel vs 371 ms serial). So + // instead take it off the critical path entirely: release on a background + // thread while the main thread proceeds. Caller must hand over EXCLUSIVE + // ownership -- no reference to the arena (or memory inside it) may survive + // the thread_launch. + ProfBeginFunction(); + U64 begin_us = now_time_us(); + Arena *arena = raw_arena; + + U64 committed_size = 0; + for (Arena *n = arena->current; n != 0; n = n->prev) { committed_size += n->cmt; } +#if ARENA_FREE_LIST + for (Arena *n = arena->free_last; n != 0; n = n->prev) { committed_size += n->cmt; } +#endif + + arena_release(arena); + + lnk_log(LNK_Log_Timers, "[teardown] background release of %llu MiB arena took %.2f ms (off main thread)", + committed_size / MB(1), (F64)(now_time_us() - begin_us) / 1000.0); + ProfEnd(); +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Fault-storm mitigation: batched PrefetchVirtualMemory over mapped input +// ranges. The .debug$S/$T parse and type-merge loops first-touch tens of GB of +// memory-mapped obj sections one 4K page fault at a time; with ~100+ links in +// flight on a build farm those per-page traps saturate the kernel machine-wide +// (prod, 126 concurrent links: dbg phase 682s kernel vs 303s user, 42M faults +// in mcvi alone). PrefetchVirtualMemory populates the ranges in bulk (large MM +// batches, no per-page trap), so issue it over each phase's input ranges right +// before the parse walk. Purely a paging hint: no output byte depends on it, +// failure is silently ignored (pre-Win8 OS / memory pressure), and prefetching +// an already-resident page is a cheap no-op -- so no residency tracking, +// blanket prefetch per phase. + +#if OS_WINDOWS +// declared locally so we do not depend on the SDK's _WIN32_WINNT gate for +// WIN32_MEMORY_RANGE_ENTRY; layout matches memoryapi.h exactly +typedef struct LNK_Win32MemoryRangeEntry +{ + void *VirtualAddress; + SIZE_T NumberOfBytes; +} LNK_Win32MemoryRangeEntry; +typedef BOOL LNK_Win32PrefetchVirtualMemoryFunc(HANDLE process, ULONG_PTR count, LNK_Win32MemoryRangeEntry *ranges, ULONG flags); // WINAPI omitted: x64-only convention + +// entries per task: the kernel's per-page population work dominates the +// syscall overhead, so small batches fanned out over the pool parallelize the +// MM work (14 GiB of mcvi input: ~0.9 s serial -> a wide parallel burst) +#define LNK_PREFETCH_BATCH_SIZE 256 + +typedef struct +{ + LNK_Win32PrefetchVirtualMemoryFunc *proc; + U64 entry_count; + LNK_Win32MemoryRangeEntry *entries; +} LNK_PrefetchTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_prefetch_task) +{ + LNK_PrefetchTask *task = raw_task; + U64 lo = task_id * LNK_PREFETCH_BATCH_SIZE; + U64 hi = Min(lo + LNK_PREFETCH_BATCH_SIZE, task->entry_count); + if (lo < hi) { + task->proc(GetCurrentProcess(), (ULONG_PTR)(hi - lo), task->entries + lo, 0); + } +} +#endif + +internal void +lnk_prefetch_ranges(TP_Context *tp, U64 range_count, Rng1U64 *ranges) +{ +#if OS_WINDOWS + // resolve once (Win8+; on older OS fall through silently). Only called from + // serial phase-setup code, so the local_persist init has no race. + local_persist LNK_Win32PrefetchVirtualMemoryFunc *prefetch_proc = 0; + local_persist B32 prefetch_proc_resolved = 0; + if (!prefetch_proc_resolved) { + prefetch_proc_resolved = 1; + HMODULE kernel32 = GetModuleHandleW(L"kernel32.dll"); + if (kernel32 != 0) { + prefetch_proc = (LNK_Win32PrefetchVirtualMemoryFunc *)GetProcAddress(kernel32, "PrefetchVirtualMemory"); + } + } + if (prefetch_proc == 0 || range_count == 0) { return; } + + Temp scratch = scratch_begin(0,0); + + // Coalesce page-aligned neighbors with a single linear pass: ranges arrive + // obj-by-obj in file-offset order, so adjacent sections of the same mapped + // obj (the common case by far) fold into one entry. No sort -- the API does + // not require ordered or disjoint ranges, overlap just costs a cheap re-walk. + LNK_Win32MemoryRangeEntry *entries = push_array_no_zero(scratch.arena, LNK_Win32MemoryRangeEntry, range_count); + U64 entry_count = 0; + U64 pending_min = 0, pending_max = 0; + for EachIndex(range_idx, range_count) { + if (ranges[range_idx].min >= ranges[range_idx].max) { continue; } + U64 min = AlignDownPow2(ranges[range_idx].min, KB(4)); + U64 max = AlignPow2 (ranges[range_idx].max, KB(4)); + if (pending_max != 0 && min <= pending_max && max >= pending_min) { + pending_min = Min(pending_min, min); + pending_max = Max(pending_max, max); + continue; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + pending_min = min; + pending_max = max; + } + if (pending_max != 0) { + entries[entry_count].VirtualAddress = (void *)pending_min; + entries[entry_count].NumberOfBytes = (SIZE_T)(pending_max - pending_min); + entry_count += 1; + } + + // fan the batches out over the pool: population is per-page kernel work, so + // this turns a serial ~1 s stall into a wide parallel burst. Purely advisory + // syscalls with no output -- any batch interleaving is fine. + LNK_PrefetchTask task = { .proc = prefetch_proc, .entry_count = entry_count, .entries = entries }; + U64 batch_count = CeilIntegerDiv(entry_count, LNK_PREFETCH_BATCH_SIZE); + if (tp != 0 && batch_count > 1) { + tp_for_parallel(tp, 0, batch_count, lnk_prefetch_task, &task); + } else { + for EachIndex(batch_idx, batch_count) { lnk_prefetch_task(0, 0, batch_idx, &task, 0); } + } + + scratch_end(scratch); +#endif +} + internal void lnk_discard_cv_debug_info(LNK_CodeViewInput *input, U64 obj_idx) { @@ -55,6 +206,32 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_s_task) lnk_error_obj(LNK_Warning_IllData, task->obj_arr[obj_idx], ".debug$S has %u file checksum sub-sections defined, picking first sub-section", checksum_data_list.node_count); } } + + // ICF-folded functions' associated .debug$S (dead-stripped, excluded from the list above): + // merge ONLY their Lines subsections. The reloc patcher patched them to the fold leader's RVA, + // so source breakpoints on folded bodies bind; symbol records stay dropped (that is the bulk + // of link.exe's size cost for the same feature). File ids in these Lines index the obj-wide + // FILECHKSMS merged above, so they stay consistent within this module. Mark 2 (fold joins a + // different source location and has locals) keeps the WHOLE record tree instead, so the watch + // window labels a folded frame with that source's own variable names. + { + LNK_Obj *obj = task->obj_arr[obj_idx]; + if (obj->icf_lines_only != 0) { + for EachIndex(sect_idx, obj->header.section_count_no_null) { + if (!obj->icf_lines_only[sect_idx]) { continue; } + LNK_ObjSection section = lnk_obj_section_from_sect_idx(obj, sect_idx); + String8 raw_data = lnk_obj_get_sect_data(obj, sect_idx, section.frange); + CV_DebugS ds = cv_debug_s_from_data(arena, raw_data); + if (obj->icf_lines_only[sect_idx] == 2) { + cv_debug_s_concat_in_place(debug_s, &ds); + } else { + String8List lines = cv_sub_section_from_debug_s(ds, CV_C13SubSectionKind_Lines); + String8List *dst = cv_sub_section_ptr_from_debug_s(debug_s, CV_C13SubSectionKind_Lines); + str8_list_concat_in_place(dst, &lines); + } + } + } + } } internal int @@ -79,7 +256,7 @@ THREAD_POOL_TASK_FUNC(lnk_parse_debug_h_task) if (obj->debug_h_sect_idx < obj->header.section_count_no_null) { LNK_ObjSection section = lnk_obj_section_from_sect_idx(obj, obj->debug_h_sect_idx); - String8 raw_debug_h = str8_substr(obj->data, section.frange); + String8 raw_debug_h = lnk_obj_get_sect_data(obj, obj->debug_h_sect_idx, section.frange); CV_DebugH *debug_h = &task->debug_h_arr[obj_idx]; LLVM_GHash ghash = {0}; U64 ghash_read_size = str8_deserial_read_struct(raw_debug_h, 0, &ghash); @@ -646,6 +823,786 @@ lnk_rrt_array_from_config(Arena *arena, LNK_Config *config) return rrt_arr; } +//////////////////////////////// +// IFC header-unit debug-record resolution + +typedef struct LNK_IfcMapEntry +{ + String8 ifc_path; // absolute .ifc path + U64 blob_slot; // resolved blob slot + 1; 0 = not yet resolved (filled by the serial + // discovery replay in lnk_apply_ifc_debug_records, memoizes path lookups) +} LNK_IfcMapEntry; + +// Parse the trivial /ifcMap TOML by hand: +// [[header-unit]] +// name = ["quote", ''] +// ifc = "" +// Registers basename(header-unit-path) -> .ifc path in `hm` (hash_map of path->raw LNK_IfcMapEntry*). +internal void +lnk_parse_ifc_map_toml(Arena *arena, HashMap *hm, String8 toml_data) +{ + U64 cursor = 0; + String8 cur_name = {0}; + while (cursor < toml_data.size) { + // read a line + U64 line_end = cursor; + while (line_end < toml_data.size && toml_data.str[line_end] != '\n') { line_end += 1; } + String8 line = str8_skip_chop_whitespace(str8_substr(toml_data, r1u64(cursor, line_end))); + cursor = line_end + 1; + + if (line.size == 0 || line.str[0] == '#') { continue; } + + if (str8_match(str8_prefix(line, 4), str8_lit("name"), 0)) { + // name = ["quote", ''] -- extract the last single-quoted token + U64 q0 = str8_find_needle(line, 0, str8_lit("'"), 0); + if (q0 < line.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("'"), 0); + if (q1 < line.size) { + cur_name = str8_substr(line, r1u64(q0 + 1, q1)); + } + } + } else if (str8_match(str8_prefix(line, 3), str8_lit("ifc"), 0)) { + U64 q0 = str8_find_needle(line, 0, str8_lit("\""), 0); + if (q0 < line.size && cur_name.size) { + U64 q1 = str8_find_needle(line, q0 + 1, str8_lit("\""), 0); + if (q1 < line.size) { + String8 ifc_path = str8_substr(line, r1u64(q0 + 1, q1)); + // key by basename of the header-unit path (matches LF_IFC_RECORD header_unit_path basename) + String8 base = str8_skip_last_slash(cur_name); + // header-unit paths use backslashes; normalize to last path component + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + LNK_IfcMapEntry *e = push_array(arena, LNK_IfcMapEntry, 1); + e->ifc_path = push_str8_copy(arena, ifc_path); + hash_map_push_string_raw(arena, hm, push_str8_copy(arena, base), e); + } + } + cur_name = str8_zero(); + } + } +} + +// Reads every /ifcMap toml, materializes the union of header-unit basename -> .ifc path. +internal HashMap +lnk_build_ifc_map(Arena *arena, LNK_Config *config) +{ + HashMap hm = {0}; + Temp scratch = scratch_begin(&arena, 1); + for EachNode(n, String8Node, config->ifc_map_list.first) { + String8 toml = lnk_read_data_from_file_path(scratch.arena, 0, n->string); + if (toml.size == 0) { + lnk_error(LNK_Error_Cmdl, "/ifcMap: unable to read TOML '%S'", n->string); + continue; + } + lnk_parse_ifc_map_toml(arena, &hm, toml); + } + scratch_end(scratch); + return hm; +} + +// LF_IFC_RECORD (0x1522) body layout (header {len,kind} already stripped from leaf.data): +// u16 version (==2); u32 ifc_type_index X; u8[16] guid; u8[16] hash; char[] header_unit_path NUL +typedef struct LNK_IfcRecord +{ + U32 ifc_type_index; // X: TI into the .ifc debug-records blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + String8 header_unit_path; + B32 is_valid; +} LNK_IfcRecord; + +internal LNK_IfcRecord +lnk_parse_ifc_record(String8 leaf_data) +{ + LNK_IfcRecord rec = {0}; + if (leaf_data.size < 2 + 4 + 16 + 16) { return rec; } + U64 off = 0; + U16 version; off += str8_deserial_read_struct(leaf_data, off, &version); + off += str8_deserial_read_struct(leaf_data, off, &rec.ifc_type_index); + MemoryCopy(rec.guid, leaf_data.str + off, 16); off += 16; + MemoryCopy(rec.hash, leaf_data.str + off, 16); off += 16; + rec.header_unit_path = str8_cstring_capped(leaf_data.str + off, leaf_data.str + leaf_data.size); + rec.is_valid = 1; + return rec; +} + +// Per-blob closure + NOTYPE-prune (third pass of lnk_apply_ifc_debug_records). Each blob is fully +// independent: it reads/writes only its own ref_bits[blob_i] and its own blob DebugT leaves, so the +// work parallelizes across the ~13 blobs with no shared state. The worklist scratch comes from the +// per-worker arena. Closure counts are written per-blob and summed afterward (order-independent). +// Open-addressing U64 hash set keyed by unique_name hash. Used to record which UDT unique_names +// already have a COMPLETE definition in a non-blob (consuming) obj, so the blob prune can keep a +// blob's complete definition only for names that NO normal obj completes (blob-only types). cap is +// a power of two; 0-hash is reserved as the empty sentinel (we OR in a bit so a real 0 can't occur). +typedef struct LNK_U64Set { U64 *slots; U64 cap; } LNK_U64Set; + +internal U64 +lnk_uname_hash(String8 s) +{ + U64 h = 5381; + for EachIndex(c, s.size) { h = ((h << 5) + h) ^ (U64)s.str[c]; } + return h | 1; // never 0 (0 is the empty sentinel) +} + +internal void +lnk_u64set_add(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { set->slots[i] = h; return; } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); + } +} + +// Thread-safe insert mirroring lnk_icf_map_put_atomic (lnk.c). The empty sentinel is 0 (not +// LNK_ICF_EMPTY); h is guaranteed nonzero by lnk_uname_hash (`| 1`) so a real key can never be 0. +// Claim an empty slot with an atomic CAS 0->h; the CAS winner owns it. On a lost race re-read the +// same slot (it may now hold our h via a duplicate, or another key) before advancing. Duplicate +// keys across objs are legal and idempotent (a slot already holding h returns), so any insertion +// order yields the identical final membership -- the set is read-only (lnk_u64set_has) afterward. +internal void +lnk_u64set_add_atomic(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { + if (ins_atomic_u64_eval_cond_assign(&set->slots[i], h, 0) == 0) { return; } + continue; + } + if (set->slots[i] == h) { return; } + i = (i + 1) & (set->cap - 1); + } +} + +internal B32 +lnk_u64set_has(LNK_U64Set *set, U64 h) +{ + U64 i = h & (set->cap - 1); + for (;;) { + if (set->slots[i] == 0) { return 0; } + if (set->slots[i] == h) { return 1; } + i = (i + 1) & (set->cap - 1); + } +} + +// Parallel collect of complete-definition unique_name hashes per non-blob obj. Each obj is scanned +// independently (read-only over its .debug$T leaves) and emits its hashes into a per-obj list; a +// serial pass then adds them to the shared open-addressing set. Moves the ~1.25s serial cv_get_udt_info +// scan off the main thread. Determinism: set membership is order-independent, serial-add reproduces. +typedef struct LNK_IfcCompleteScanTask +{ + LNK_CodeViewInput *input; + U64 **out_hashes; // per-obj hash array (allocated by task) + U64 *out_counts; // per-obj count +} LNK_IfcCompleteScanTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_complete_scan_task) +{ + LNK_IfcCompleteScanTask *task = raw_task; + U64 obj_idx = task_id; + CV_DebugT *dt = &task->input->debug_t_arr[obj_idx]; + U64 *hashes = push_array_no_zero(arena, U64, dt->count ? dt->count : 1); + U64 n = 0; + for EachIndex(leaf_idx, dt->count) { + CV_Leaf leaf = cv_debug_t_get_leaf(dt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } + hashes[n++] = lnk_uname_hash(ui.unique_name); + } + task->out_hashes[obj_idx] = hashes; + task->out_counts[obj_idx] = n; +} + +// Parallel merge of the per-obj complete-def hash lists into the shared set. Replaces the serial +// lnk_u64set_add loop (the ~914ms hotspot -- 801ms of it first-touch KiPageFault on a single thread +// faulting a 128MB+ set). lnk_u64set_add_atomic spreads both the random-scatter probes AND the +// page faults across the pool. Determinism: keys may legitimately duplicate across objs, but the +// insert is idempotent and the set is read-only afterward (lnk_u64set_has), so insertion order +// cannot change the final membership -- output is bit-identical to the serial merge. +typedef struct LNK_IfcSetMergeTask +{ + Rng1U64 *ranges; + U64 **out_hashes; + U64 *out_counts; + LNK_U64Set *set; + U64 nonblob_count; +} LNK_IfcSetMergeTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_set_merge_task) +{ + LNK_IfcSetMergeTask *task = raw_task; + for EachInRange(obj_idx, task->ranges[task_id]) { + U64 *h = task->out_hashes[obj_idx]; + U64 n = task->out_counts[obj_idx]; + for EachIndex(t, n) { lnk_u64set_add_atomic(task->set, h[t]); } + } +} + +// Fused discovery+redirect scan (passes 1+2 of lnk_apply_ifc_debug_records): each consuming +// obj's .debug$T is swept ONCE, in parallel, for 0x1522 (LF_IFC_RECORD) leaves. The worker +// parses each record, resolves its header-unit basename against the read-only ifc_map_hm, and +// emits one raw record per 0x1522 leaf in ascending leaf_idx order. Workers write NOTHING +// (no NOTYPE, no discovery, no redirects): every order-sensitive effect -- .ifc first-encounter +// slot assignment, NOTYPE rewrites, redirect hash-map push order, ref_bits seeding -- is +// replayed SERIALLY from these records in ascending obj_idx, then ascending leaf_idx: the exact +// order of the original serial passes, so output is byte-for-byte identical. +typedef struct LNK_IfcRawRec +{ + U64 leaf_idx; // 0x1522 leaf index inside the consuming obj + CV_TypeIndex K; // consuming obj's local placeholder TI + U32 ifc_type_index; // X: TI into the .ifc blob (base 0x1000) + U8 guid[16]; + U8 hash[16]; + LNK_IfcMapEntry *entry; // basename -> map entry (0: invalid record or no map hit) + U64 blob_i_plus1; // resolved blob slot + 1 (serial replay fills; 0 = unresolved) + U64 blob_leaf_idx; // resolved leaf inside the blob (serial replay fills) +} LNK_IfcRawRec; + +typedef struct LNK_IfcScanTask +{ + LNK_CodeViewInput *input; + HashMap *ifc_map_hm; // read-only in workers + LNK_IfcRawRec **out_recs; // per-obj ordered raw records (allocated by task) + U64 *out_counts; // per-obj record count +} LNK_IfcScanTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_scan_task) +{ + LNK_IfcScanTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + + // count 0x1522 leaves first to size the per-obj record array + U64 ifc_leaf_count = 0; + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind == 0x1522) { ifc_leaf_count += 1; } + } + if (ifc_leaf_count == 0) { task->out_recs[obj_idx] = 0; task->out_counts[obj_idx] = 0; return; } + + LNK_IfcRawRec *recs = push_array_no_zero(arena, LNK_IfcRawRec, ifc_leaf_count); + U64 n = 0; + + for EachIndex(leaf_idx, debug_t->count) { + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + if (hdr->kind != 0x1522) { continue; } + + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + LNK_IfcRecord rec = lnk_parse_ifc_record(leaf.data); + + LNK_IfcRawRec *r = &recs[n++]; + r->leaf_idx = leaf_idx; + r->K = cv_ti_from_leaf_idx(debug_t, CV_TypeIndexSource_TPI, leaf_idx); + r->ifc_type_index = rec.ifc_type_index; + r->entry = 0; + r->blob_i_plus1 = 0; + r->blob_leaf_idx = 0; + + if (rec.is_valid) { + MemoryCopy(r->guid, rec.guid, 16); + MemoryCopy(r->hash, rec.hash, 16); + String8 base = str8_skip_last_slash(rec.header_unit_path); + U64 bs = str8_find_needle_reverse(base, 0, str8_lit("\\"), 0); + if (bs) { base = str8_skip(base, bs); } + r->entry = hash_map_search_string_raw(task->ifc_map_hm, base); + } + } + + task->out_recs[obj_idx] = recs; + task->out_counts[obj_idx] = n; +} + +// Parallel per-obj record resolution + placeholder NOTYPE (runs after discovery/read/injection, +// when entry->blob_slot, ifc_files, and the injected blob debug_t entries are all frozen/read-only). +// Each worker fills its own obj's raw records in place (blob_i_plus1/blob_leaf_idx), rewrites its +// own 0x1522 leaves to NOTYPE (per-obj disjoint, constant value -- order-free), and reports the +// resolved count + K range. The serial replay below then only pushes redirects in the original +// (obj_idx, leaf_idx) order, so hash-map push order and all outputs stay bit-identical. +typedef struct LNK_IfcResolveTask +{ + LNK_CodeViewInput *input; + IFC_File *ifc_files; + LNK_IfcRawRec **recs; // per-obj raw records from the scan + U64 *counts; // per-obj record count + U64 *res_counts; // out: per-obj resolved record count + U64 *k_first; // out: first resolved K (valid when res_counts != 0) + U64 *k_last; // out: last resolved K (valid when res_counts != 0) +} LNK_IfcResolveTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_resolve_task) +{ + LNK_IfcResolveTask *task = raw_task; + U64 obj_idx = task_id; + LNK_CodeViewInput *input = task->input; + LNK_IfcRawRec *recs = task->recs[obj_idx]; + U64 n = task->counts[obj_idx]; + if (n == 0) { task->res_counts[obj_idx] = 0; return; } + CV_DebugT *debug_t = &input->debug_t_arr[obj_idx]; + + U64 res_count = 0; + U64 k_first = 0, k_last = 0; + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(debug_t, r->leaf_idx); + // exclude the placeholder leaf from output regardless: rewrite to NOTYPE + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + if (r->entry == 0 || r->entry->blob_slot == 0) { continue; } + U64 blob_i = r->entry->blob_slot - 1; + IFC_File *f = &task->ifc_files[blob_i]; + B32 hash_ok = MemoryMatch(r->guid, f->content_hash, 16) && + MemoryMatch(r->hash, f->content_hash + 16, 16); + if (!f->is_valid || !hash_ok) { continue; } + CV_DebugT *bdt = &input->debug_t_arr[input->ifc_obj_range.min + blob_i]; + U64 blob_leaf_idx = cv_leaf_idx_from_ti(bdt, CV_TypeIndexSource_TPI, r->ifc_type_index); + if (blob_leaf_idx >= bdt->count) { continue; } + r->blob_i_plus1 = blob_i + 1; + r->blob_leaf_idx = blob_leaf_idx; + if (res_count == 0) { k_first = r->K; } + k_last = r->K; + res_count += 1; + } + task->res_counts[obj_idx] = res_count; + task->k_first[obj_idx] = k_first; + task->k_last[obj_idx] = k_last; +} + +// Parallel .ifc read + `.msvc.trait.debug-records` parse into PRE-ASSIGNED slots. Slot order +// (== blob obj order == output order) is fixed by the serial discovery replay before any file +// is read, so going wide here cannot reorder anything. Workers do not call lnk_error: read +// failures are collected per slot and reported serially in slot order afterward (identical +// message order to the old serial read; LNK_Error_Cmdl stops the link either way). Worker-arena +// allocations (file bytes + leaf offsets) are long-lived, same as the parallel .debug$T parse +// (lnk_parse_debug_t_task pattern). +typedef struct LNK_IfcReadTask +{ + String8 *paths; // per-slot .ifc path + IFC_File *ifc_files; // per-slot output + CV_DebugT *blob_debug_t; // per-slot output + String8 *errors; // per-slot read error (size 0 = ok) +} LNK_IfcReadTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_read_task) +{ + LNK_IfcReadTask *task = raw_task; + U64 slot = task_id; + String8 err = {0}; + IFC_File f = ifc_file_read(arena, task->paths[slot], &err); + task->ifc_files[slot] = f; + task->errors[slot] = err; + if (f.is_valid) { + // parse the raw CV leaf stream (no signature, TI base 0x1000) + task->blob_debug_t[slot] = cv_debug_t_from_data(arena, f.debug_records, 1); + } else { + MemoryZeroStruct(&task->blob_debug_t[slot]); + } +} + +typedef struct LNK_IfcCloseTask +{ + LNK_CodeViewInput *input; + U8 **ref_bits; + U64 *closure_leaves; // per-blob output: # leaves surviving in closure + LNK_U64Set *nonblob_complete; // unique_name hashes completed by some non-blob obj +} LNK_IfcCloseTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_ifc_close_blob_task) +{ + LNK_IfcCloseTask *task = raw_task; + U64 blob_i = task_id; + LNK_CodeViewInput *input = task->input; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + CV_DebugT *bdt = &input->debug_t_arr[blob_obj_idx]; + U8 *bits = task->ref_bits[blob_i]; + U64 closure = 0; + if (bdt->count == 0) { task->closure_leaves[blob_i] = 0; return; } + + Temp wtemp = temp_begin(arena); + + // Extra closure roots for forward-ref completion: in CodeView a forward-ref UDT is completed by + // ANY same-unique_name complete definition in the PDB. A consuming obj typically emits only a + // forward-ref of a header-unit type; the full-merge build incidentally kept the matching complete + // definition from the .ifc blob, so the debugger could complete it. On-demand would drop that + // definition (nothing references it by TI), leaving the type incomplete vs full-merge. To preserve + // fidelity WITHOUT dragging the whole blob, root every blob complete-def UDT whose unique_name has + // NO complete definition in any non-blob obj (i.e. blob-only types -- trait/delegate marker structs + // etc.). Common types (FString, FGuid, ...) are completed by normal objs, so their redundant blob + // copies stay pruned. Members of the kept defs are pulled by the closure walk below. + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { continue; } // already a root + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (!(ui.props & CV_TypeProp_HasUniqueName) || ui.unique_name.size == 0) { continue; } + if (ui.props & CV_TypeProp_FwdRef) { continue; } // only complete definitions + U64 h = lnk_uname_hash(ui.unique_name); + if (lnk_u64set_has(task->nonblob_complete, h)) { continue; } // a normal obj already completes it + bits[leaf_idx >> 3] |= (U8)(1u << (leaf_idx & 7)); + } + + U64 *worklist = push_array_no_zero(wtemp.arena, U64, bdt->count); + U64 wl_count = 0; + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { worklist[wl_count++] = leaf_idx; } + } + + while (wl_count) { + U64 leaf_idx = worklist[--wl_count]; + CV_Leaf leaf = cv_debug_t_get_leaf(bdt, leaf_idx); + Temp itemp = temp_begin(wtemp.arena); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(itemp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(leaf.data, ti_info.offset, sizeof(*ti_ptr)); + if (ti_ptr == 0) { continue; } + CV_TypeIndex sub_ti = memory_read32(ti_ptr); + if (sub_ti < bdt->ti_ranges[ti_info.source].min || + sub_ti >= bdt->ti_ranges[ti_info.source].max) { continue; } + U64 sub_leaf_idx = cv_leaf_idx_from_ti(bdt, ti_info.source, sub_ti); + if (sub_leaf_idx >= bdt->count) { continue; } + if (bits[sub_leaf_idx >> 3] & (1u << (sub_leaf_idx & 7))) { continue; } + bits[sub_leaf_idx >> 3] |= (U8)(1u << (sub_leaf_idx & 7)); + worklist[wl_count++] = sub_leaf_idx; + } + temp_end(itemp); + } + temp_end(wtemp); + + for EachIndex(leaf_idx, bdt->count) { + if (bits[leaf_idx >> 3] & (1u << (leaf_idx & 7))) { closure += 1; continue; } + CV_LeafHeader *hdr = cv_debug_t_get_leaf_header(bdt, leaf_idx); + if (hdr->kind == CV_LeafKind_NOTYPE) { continue; } + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, kind), (U16)CV_LeafKind_NOTYPE); + memory_write16(MemberFromPtr(CV_LeafHeader, hdr, size), (U16)sizeof(CV_LeafKind)); + } + task->closure_leaves[blob_i] = closure; +} + +// Injects referenced .ifc debug-records blobs as extra "objs" in `input`, scans every +// consuming obj's .debug$T for LF_IFC_RECORD (0x1522) leaves, registers each placeholder +// local TI -> blob leaf redirect, and rewrites the 0x1522 leaf to NOTYPE so it is excluded +// from the output TPI. Must run after .debug$T is parsed and before min-type-index / symbol +// setup (which iterate input->count). +internal void +lnk_apply_ifc_debug_records(TP_Context *tp, TP_Arena *tp_arena, LNK_CodeViewInput *input, LNK_Config *config) +{ + ProfBeginFunction(); + U64 apply_begin_us = now_time_us(); + Temp scratch = scratch_begin(&tp_arena->v[0], 1); + Arena *arena = tp_arena->v[0]; + + // basename -> .ifc path + HashMap ifc_map_hm = lnk_build_ifc_map(scratch.arena, config); + U64 discover_begin_us = now_time_us(); + + // --- fused scan (old passes 1+2, parallel): ONE sweep of every consuming obj's .debug$T + // emits per-obj raw 0x1522 records in ascending leaf_idx order (record parse + basename -> + // ifc_map_hm entry resolution happen in the workers; nothing is written). Every + // order-sensitive effect is replayed serially from these records below. --- + LNK_IfcScanTask scan = {0}; + scan.input = input; + scan.ifc_map_hm = &ifc_map_hm; + scan.out_recs = push_array(scratch.arena, LNK_IfcRawRec *, input->obj_count); + scan.out_counts = push_array(scratch.arena, U64, input->obj_count); + tp_for_parallel(tp, tp_arena, input->obj_count, lnk_ifc_scan_task, &scan); + U64 scan_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] parallel scan in %.2f ms", (F64)(scan_end_us - discover_begin_us) / 1000.0); + + // --- serial discovery replay: assign .ifc blob slots in first-encounter order (ascending + // obj_idx, then ascending leaf_idx -- identical to the old serial pass) WITHOUT reading any + // file, so the reads can go wide below. De-dup by path; entry->blob_slot memoizes the path + // lookup. 256-slot cap semantics preserved: on overflow the entry stays unresolved and its + // records never redirect. --- + HashMap ifc_path_to_blobidx = {0}; // path -> (blob slot index + 1) + IFC_File *ifc_files = push_array(scratch.arena, IFC_File, 256); + U64 ifc_file_count = 0; + CV_DebugT blob_debug_t[256] = {0}; + String8 slot_paths[256] = {0}; + for EachIndex(obj_idx, input->obj_count) { + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + for EachIndex(t, n) { + LNK_IfcMapEntry *e = recs[t].entry; + if (e == 0 || e->blob_slot) { continue; } + U64 *slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + if (slot == 0) { + if (ifc_file_count >= 256) { continue; } + hash_map_push_string_u64(scratch.arena, &ifc_path_to_blobidx, e->ifc_path, ifc_file_count + 1); + slot_paths[ifc_file_count] = e->ifc_path; + ifc_file_count += 1; + slot = hash_map_search_string_u64(&ifc_path_to_blobidx, e->ifc_path); + } + e->blob_slot = *slot; + } + } + + if (ifc_file_count == 0) { goto done; } + U64 discover_end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "[IFC] discover replay in %.2f ms", (F64)(discover_end_us - scan_end_us) / 1000.0); + + // --- parallel .ifc read + debug-records parse into the pre-assigned slots; report read + // errors serially in slot order (identical message order to the old serial read). --- + { + LNK_IfcReadTask read = {0}; + read.paths = slot_paths; + read.ifc_files = ifc_files; + read.blob_debug_t = blob_debug_t; + read.errors = push_array(scratch.arena, String8, ifc_file_count); + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_read_task, &read); + for EachIndex(i, ifc_file_count) { + if (!ifc_files[i].is_valid) { lnk_error(LNK_Error_Cmdl, "/ifcDebugRecords: %S", read.errors[i]); } + } + lnk_log(LNK_Log_Timers, "[IFC] read+parse %llu blob(s) in %.2f ms", ifc_file_count, (F64)(now_time_us() - discover_end_us) / 1000.0); + } + + // --- inject blob objs into the parallel arrays (like type servers, but in ifc_obj_range) --- + U64 prev_count = input->count; + U64 new_count = prev_count + ifc_file_count; + + LNK_Obj **obj_arr2 = push_array(arena, LNK_Obj *, new_count); + CV_DebugS *debug_s_arr2 = push_array(arena, CV_DebugS, new_count); + CV_DebugT *debug_t_arr2 = push_array(arena, CV_DebugT, new_count); + CV_DebugH *debug_h_arr2 = push_array(arena, CV_DebugH, new_count); + U64 *obj_to_ts2 = push_array(arena, U64, new_count); + + MemoryCopyTyped(obj_arr2, input->obj_arr, prev_count); + MemoryCopyTyped(debug_s_arr2, input->debug_s_arr, prev_count); + MemoryCopyTyped(debug_t_arr2, input->debug_t_arr, prev_count); + MemoryCopyTyped(debug_h_arr2, input->debug_h_arr, prev_count); + MemoryCopyTyped(obj_to_ts2, input->obj_to_ts, prev_count); + MemorySet(obj_to_ts2 + prev_count, 0xff, ifc_file_count * sizeof(U64)); // blobs are not type servers + + // blob obj indices + index list for hash-deep / dedup + U32Array ifc_indices = { .v = push_array(arena, U32, ifc_file_count) }; + for EachIndex(i, ifc_file_count) { + U64 blob_obj_idx = prev_count + i; + LNK_Obj *blob_obj = push_array(arena, LNK_Obj, 1); + blob_obj->path = ifc_files[i].path; + obj_arr2[blob_obj_idx] = blob_obj; + debug_t_arr2[blob_obj_idx] = blob_debug_t[i]; + ifc_indices.v[ifc_indices.count++] = (U32)blob_obj_idx; + } + + input->count = new_count; + input->obj_arr = obj_arr2; + input->debug_s_arr = debug_s_arr2; + input->debug_t_arr = debug_t_arr2; + input->debug_h_arr = debug_h_arr2; + input->obj_to_ts = obj_to_ts2; + input->ifc_obj_range = r1u64(prev_count, new_count); + input->ifc_indices = ifc_indices; // hashed + deduped before int objs (see lnk_merge_types) + + // --- on-demand pruning state: per blob, a "referenced" bitset of leaf indices that + // are reachable from some consuming obj's 0x1522 redirect (the closure roots). Only these + // + their transitive blob-internal deps get merged; the rest are rewritten to NOTYPE so the + // hash/dedup pipeline skips ~all of the ~1.5M blob leaves that nothing references. --- + U8 **ref_bits = push_array(scratch.arena, U8 *, ifc_file_count); + for EachIndex(i, ifc_file_count) { + U64 c = blob_debug_t[i].count; + ref_bits[i] = push_array(scratch.arena, U8, (c + 7) / 8); // zero-init -> nothing referenced yet + } + + // --- parallel record resolution + placeholder NOTYPE: per-obj disjoint, order-free (see + // lnk_ifc_resolve_task). All inputs (entry->blob_slot, ifc_files, blob debug_t) are frozen + // after the discovery/read/injection steps above. --- + input->has_ifc_redirects = 1; + input->ifc_redirect_bits = push_array(arena, U64 *, input->count); + input->ifc_redirect_ti_rng = push_array(arena, Rng1U64, input->count); + U64 redirect_count = 0; + U64 resolve_begin_us = now_time_us(); + LNK_IfcResolveTask resolve = {0}; + resolve.input = input; + resolve.ifc_files = ifc_files; + resolve.recs = scan.out_recs; + resolve.counts = scan.out_counts; + resolve.res_counts = push_array(scratch.arena, U64, input->obj_count); + resolve.k_first = push_array(scratch.arena, U64, input->obj_count); + resolve.k_last = push_array(scratch.arena, U64, input->obj_count); + tp_for_parallel(tp, 0, input->obj_count, lnk_ifc_resolve_task, &resolve); + lnk_log(LNK_Log_Timers, "[IFC] parallel resolve in %.2f ms", (F64)(now_time_us() - resolve_begin_us) / 1000.0); + + // --- serial redirect replay: push redirects in ascending obj_idx, then ascending leaf_idx -- + // the exact original serial order -- so the redirect hash-map push order, ref_bits seeding, + // and redirect_count are bit-for-bit identical to the serial code. + U64 merge_begin_us = now_time_us(); + for EachIndex(obj_idx, input->obj_count) { + LNK_IfcRawRec *recs = scan.out_recs[obj_idx]; + U64 n = scan.out_counts[obj_idx]; + if (n == 0 || resolve.res_counts[obj_idx] == 0) { continue; } + + // exact key filter range: records are K-ascending (the scan emits leaf_idx ascending and + // cv_ti_from_leaf_idx is monotonic), so [k_first, k_last] spans all resolved keys. + Rng1U64 krng = r1u64(resolve.k_first[obj_idx], resolve.k_last[obj_idx] + 1); + input->ifc_redirect_ti_rng[obj_idx] = krng; + input->ifc_redirect_bits[obj_idx] = push_array(arena, U64, (dim_1u64(krng) + 63) / 64); + for EachIndex(t, n) { + LNK_IfcRawRec *r = &recs[t]; + if (r->blob_i_plus1 == 0) { continue; } + U64 blob_i = r->blob_i_plus1 - 1; + U64 blob_obj_idx = input->ifc_obj_range.min + blob_i; + hash_map_push_u64_u64(arena, &input->ifc_redirect_hm, + Compose64Bit(obj_idx, r->K), + Compose64Bit(blob_obj_idx, r->blob_leaf_idx)); + U64 rel = r->K - krng.min; + input->ifc_redirect_bits[obj_idx][rel >> 6] |= (1ull << (rel & 63)); + redirect_count += 1; + // seed closure root: this blob leaf is referenced + ref_bits[blob_i][r->blob_leaf_idx >> 3] |= (U8)(1u << (r->blob_leaf_idx & 7)); + } + } + + lnk_log(LNK_Log_Timers, "[IFC] redirect replay in %.2f ms", (F64)(now_time_us() - merge_begin_us) / 1000.0); + + // --- third pass: per blob, close the referenced set over blob-internal sub-TIs, then + // NOTYPE every leaf not in the closure. cv_leaf_idx_from_ti on a raw blob is source-agnostic + // (source_offsets are 0, all ti_ranges == [0x1000, 0x1000+count)) so a sub-TI maps directly to + // leaf_idx = ti - 0x1000 regardless of its CV_TypeIndexSource label. Walk is iterative (worklist). + U64 total_blob_leaves = 0, total_closure_leaves = 0; + U64 closure_begin_us = now_time_us(); + for EachIndex(blob_i, ifc_file_count) { + total_blob_leaves += input->debug_t_arr[input->ifc_obj_range.min + blob_i].count; + } + if (ifc_file_count) { + // Build the set of unique_names that already have a COMPLETE definition in some non-blob obj. + // The blob prune keeps a blob complete-def only when its name is absent here (blob-only type), + // so forward-refs that no normal obj can complete still get their definition (full-merge fidelity) + // while redundant blob copies of normally-defined types stay pruned. Size to ~2x the non-blob + // complete-def count, rounded up to a power of two, for low load factor. + U64 nonblob_complete_estimate = 0; + for EachIndex(obj_idx, input->ifc_obj_range.min) { + nonblob_complete_estimate += input->debug_t_arr[obj_idx].source_counts[CV_TypeIndexSource_TPI]; + } + LNK_U64Set nonblob_complete = {0}; + nonblob_complete.cap = 1; + while (nonblob_complete.cap < (nonblob_complete_estimate * 2 + 16)) { nonblob_complete.cap <<= 1; } + nonblob_complete.slots = push_array(scratch.arena, U64, nonblob_complete.cap); + // parallel scan: each non-blob obj emits its complete-def hashes; serial merge adds to the set. + U64 nonblob_count = input->ifc_obj_range.min; + if (nonblob_count) { + LNK_IfcCompleteScanTask scan = {0}; + scan.input = input; + scan.out_hashes = push_array(scratch.arena, U64 *, nonblob_count); + scan.out_counts = push_array(scratch.arena, U64, nonblob_count); + tp_for_parallel(tp, tp_arena, nonblob_count, lnk_ifc_complete_scan_task, &scan); + // parallel atomic-CAS merge (replaces the serial lnk_u64set_add loop): output-identical + // because set membership is order-independent + idempotent (see lnk_u64set_add_atomic). + LNK_IfcSetMergeTask merge = {0}; + merge.ranges = tp_divide_work(scratch.arena, nonblob_count, tp->worker_count); + merge.out_hashes = scan.out_hashes; + merge.out_counts = scan.out_counts; + merge.set = &nonblob_complete; + merge.nonblob_count = nonblob_count; + tp_for_parallel(tp, 0, tp->worker_count, lnk_ifc_set_merge_task, &merge); + } + + LNK_IfcCloseTask close_task = {0}; + close_task.input = input; + close_task.ref_bits = ref_bits; + close_task.closure_leaves = push_array(scratch.arena, U64, ifc_file_count); + close_task.nonblob_complete = &nonblob_complete; + tp_for_parallel(tp, tp_arena, ifc_file_count, lnk_ifc_close_blob_task, &close_task); + for EachIndex(blob_i, ifc_file_count) { total_closure_leaves += close_task.closure_leaves[blob_i]; } + } + (void)tp; + lnk_log(LNK_Log_Timers, "[IFC] closure pass in %.2f ms", (F64)(now_time_us() - closure_begin_us) / 1000.0); + + lnk_log(LNK_Log_Debug, "[IFC] injected %llu .ifc blob(s), %llu record redirect(s); on-demand closure %llu / %llu blob leaves (%.1f%%)", + ifc_file_count, redirect_count, total_closure_leaves, total_blob_leaves, + total_blob_leaves ? (100.0 * (F64)total_closure_leaves / (F64)total_blob_leaves) : 0.0); + +done: + scratch_end(scratch); + lnk_log(LNK_Log_Timers, "[IFC] apply total in %.2f ms", (F64)(now_time_us() - apply_begin_us) / 1000.0); + ProfEnd(); +} + +//////////////////////////////// +// parallel setup tasks for lnk_make_code_view_input + +// Loop 3 (PCH/ext/int classification). The expensive predicates -- the read-only rrt_hm lookup +// and cv_debug_t_is_type_server_ref -- run in parallel per obj. Each obj's class tag and PCH-merge +// mutation are fully independent, so this pass is data-parallel. The ordered 3-array compaction +// and the MultipleDebugTAndDebugP warning are then replayed SERIALLY in obj_idx order, so output +// (array contents/order + warning order + discarded set) is byte-identical to the serial loop. +typedef struct LNK_CvClassifyTask +{ + LNK_CodeViewInput *input; + CV_DebugT *debug_p_arr; + HashMap *rrt_hm; // read-only after build + LNK_Obj **obj_arr; + U8 *class_tag; // 0=debug_p, 1=ext, 2=int + U8 *warn_multi; +} LNK_CvClassifyTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_classify_task) +{ + LNK_CvClassifyTask *t = raw_task; + U64 obj_idx = task_id; + CV_DebugT *debug_t = &t->input->debug_t_arr[obj_idx]; + CV_DebugT *debug_p = &t->debug_p_arr[obj_idx]; + + // classify (same predicate order/precedence as the serial loop) + U8 tag; + if (hash_map_search_path_u64(t->rrt_hm, t->obj_arr[obj_idx]->path)) { tag = 1; } + else if (debug_p->count > 0 && debug_t->count == 0) { tag = 0; } + else if (cv_debug_t_is_type_server_ref(debug_t)) { tag = 1; } + else { tag = 2; } + t->class_tag[obj_idx] = tag; + + // per-obj independent debug_t mutation (identical to serial) + if (debug_t->count == 0 && debug_p->count > 0) { + *debug_t = *debug_p; + } else if (debug_t->count && debug_p->count) { + t->warn_multi[obj_idx] = 1; // defer warning to serial obj-order replay + MemoryZeroStruct(debug_t); + MemoryZeroStruct(debug_p); + } +} + +// Loop 4 (Make Symbol Inputs) count pass. cv_sub_section_from_debug_s is a pure read of the +// already-parsed data_list, so caching each obj's Symbols sub-section list in parallel is safe. +typedef struct LNK_CvSymTask +{ + LNK_CodeViewInput *input; + String8List *per_obj_syms; + U64 *counts; // per-obj node_count (count pass) + U64 *offsets; // per-obj symbol_inputs offset (fill pass) +} LNK_CvSymTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_count_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + t->per_obj_syms[obj_idx] = cv_sub_section_from_debug_s(t->input->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); + t->counts[obj_idx] = t->per_obj_syms[obj_idx].node_count; +} + +// Loop 4 fill pass. Each obj writes a disjoint, contiguous range of symbol_inputs starting at its +// prefix-sum offset, in node order -- byte-identical to the serial append (which walked obj_idx +// ascending, each obj's nodes in list order). +internal +THREAD_POOL_TASK_FUNC(lnk_cv_sym_fill_task) +{ + LNK_CvSymTask *t = raw_task; + U64 obj_idx = task_id; + U64 cur = t->offsets[obj_idx]; + String8List s = t->per_obj_syms[obj_idx]; + for EachNode(n, String8Node, s.first) { + LNK_SymbolInput *in = &t->input->symbol_inputs[cur++]; + in->obj_idx = obj_idx; + in->raw_symbols = n->string; + } +} + internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 obj_count, LNK_Obj **obj_arr, LNK_RRT_Array rrt_input) { @@ -666,47 +1623,51 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfBegin("Apply RRT to Objs"); - // hash map (obj path, obj idx) + // hash map (obj path, obj idx). Kept SERIAL: HashMap is a 4-ary trie whose insert mutates shared + // child pointers + arena-allocates nodes -> not safe for concurrent insert. Only built (and + // consulted) when there is at least one input RRT; the monolithic Engine.dll link has none. HashMap obj_path_hm = {0}; - for EachIndex(obj_idx, obj_count) { - hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); - } + if (rrt_input.count) { + for EachIndex(obj_idx, obj_count) { + hash_map_push_path_u64(scratch.arena, &obj_path_hm, obj_arr[obj_idx]->path, obj_idx); + } - for EachIndex(obj_idx, obj_count) { - LNK_Obj *obj = obj_arr[obj_idx]; - U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); - - // obj is not part of any input RRT - if (packed_rrt_idx == 0) { continue; } - - // unpack index - U32 rrt_idx = *packed_rrt_idx >> 32; - U32 rrt_obj_idx = *packed_rrt_idx & max_U32; - LNK_RRT *rrt = &rrt_input.v[rrt_idx]; - - // obj was recompiled, do not apply RRT indirection - FileProperties obj_file_props = properties_from_file_path(obj->path); - if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } - - // invalidate debug section pointers - obj->debug_t_sect_idx = ~0; - obj->debug_p_sect_idx = ~0; - obj->debug_h_sect_idx = ~0; - - // apply type index map - obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; - obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; - - // apply PCH info - U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; - if (rrt_pch_obj_idx < rrt->obj_count) { - String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; - U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); - obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; - obj->pch_obj_idx = pch_obj_idx; - } else { - obj->pch_ti_range = r1u64(0,0); - obj->pch_obj_idx = ~0; + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + U64 *packed_rrt_idx = hash_map_search_path_u64(&rrt_hm, obj->path); + + // obj is not part of any input RRT + if (packed_rrt_idx == 0) { continue; } + + // unpack index + U32 rrt_idx = *packed_rrt_idx >> 32; + U32 rrt_obj_idx = *packed_rrt_idx & max_U32; + LNK_RRT *rrt = &rrt_input.v[rrt_idx]; + + // obj was recompiled, do not apply RRT indirection + FileProperties obj_file_props = properties_from_file_path(obj->path); + if (rrt->obj_time_stamps[rrt_obj_idx] != obj_file_props.modified) { continue; } + + // invalidate debug section pointers + obj->debug_t_sect_idx = ~0; + obj->debug_p_sect_idx = ~0; + obj->debug_h_sect_idx = ~0; + + // apply type index map + obj->ti_range = rrt->obj_ti_ranges[rrt_obj_idx]; + obj->ti_map = rrt->obj_ti_maps [rrt_obj_idx]; + + // apply PCH info + U32 rrt_pch_obj_idx = rrt->obj_pch_indices[rrt_obj_idx]; + if (rrt_pch_obj_idx < rrt->obj_count) { + String8 rrt_pch_obj_path = rrt->obj_paths.v[rrt_pch_obj_idx]; + U64 pch_obj_idx = *hash_map_search_path_u64(&obj_path_hm, rrt_pch_obj_path); + obj->pch_ti_range = rrt->obj_pch_ti_ranges[rrt_obj_idx]; + obj->pch_obj_idx = pch_obj_idx; + } else { + obj->pch_ti_range = r1u64(0,0); + obj->pch_obj_idx = ~0; + } } } ProfEnd(); @@ -715,6 +1676,50 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, input.debug_s_list_arr = lnk_collect_obj_sections(tp, tp_arena, obj_count, obj_arr, str8_lit(".debug$S"), 0); ProfEnd(); + // batch-populate the mapped .debug$S/$T/$P/$H input ranges before the parse + // loops below first-touch them page by page (see lnk_prefetch_ranges) + ProfScope("Prefetch CodeView") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 3 * obj_count; // debug$T + debug$P + debug$H + for EachIndex(obj_idx, obj_count) { range_cap += input.debug_s_list_arr[obj_idx].node_count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachIndex(obj_idx, obj_count) { + LNK_Obj *obj = obj_arr[obj_idx]; + + for EachNode(n, String8Node, input.debug_s_list_arr[obj_idx].first) { + if (n->string.size) { ranges[range_count++] = rng_1u64((U64)n->string.str, (U64)n->string.str + n->string.size); } + } + if (obj->debug_t_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_t_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_t_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (obj->debug_p_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_p_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_p_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + if (config->ghash && obj->debug_h_sect_idx < obj->header.section_count_no_null) { + LNK_ObjSection sect = lnk_obj_section_from_sect_idx(obj, obj->debug_h_sect_idx); + String8 data = lnk_obj_get_sect_data(obj, obj->debug_h_sect_idx, sect.frange); + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + Assert(range_count <= range_cap); + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, range_count, ranges); + lnk_log(LNK_Log_Timers, "[mcvi] prefetched %llu debug section ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + // profiler info if (lnk_get_log_status(LNK_Log_Debug) || PROFILE_TELEMETRY) { U64 total_debug_s_size = 0, total_debug_t_size = 0, total_debug_p_size = 0, total_debug_h_size = 0; @@ -777,14 +1782,14 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, LNK_ObjSection debug_t_sect = lnk_obj_section_from_sect_idx(obj, obj->debug_t_sect_idx); raw_debug_t_arr[obj_idx].count = 1; raw_debug_t_arr[obj_idx].v = push_array(scratch.arena, String8, 1); - raw_debug_t_arr[obj_idx].v[0] = str8_substr(obj->data, debug_t_sect.frange); + raw_debug_t_arr[obj_idx].v[0] = lnk_obj_get_sect_data(obj, obj->debug_t_sect_idx, debug_t_sect.frange); } if (obj->debug_p_sect_idx < obj->header.section_count_no_null) { LNK_ObjSection debug_p_sect = lnk_obj_section_from_sect_idx(obj, obj->debug_p_sect_idx); raw_debug_p_arr[obj_idx].count = 1; raw_debug_p_arr[obj_idx].v = push_array(scratch.arena, String8, 1); - raw_debug_p_arr[obj_idx].v[0] = str8_substr(obj->data, debug_p_sect.frange); + raw_debug_p_arr[obj_idx].v[0] = lnk_obj_get_sect_data(obj, obj->debug_p_sect_idx, debug_p_sect.frange); } } @@ -813,25 +1818,35 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfEnd(); // sort objs based on type: PCH, /Zi (external), /Z7 (internal) - input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); + input.debug_p_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.ext_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); input.int_obj_indices.v = push_array(tp_arena->v[0], U32, obj_count); - for EachIndex(obj_idx, obj_count) { - CV_DebugT *debug_t = &input.debug_t_arr[obj_idx]; - CV_DebugT *debug_p = &debug_p_arr[obj_idx]; - U32Array *arr_ptr; - if (hash_map_search_path_u64(&rrt_hm, obj_arr[obj_idx]->path)) { arr_ptr = &input.ext_obj_indices; } - else if (debug_p->count > 0 && debug_t->count == 0) { arr_ptr = &input.debug_p_indices; } - else if (cv_debug_t_is_type_server_ref(debug_t)) { arr_ptr = &input.ext_obj_indices; } - else { arr_ptr = &input.int_obj_indices; } - arr_ptr->v[arr_ptr->count++] = obj_idx; + ProfScope("Classify Objs") + { + // parallel: classify each obj + apply per-obj debug_t mutation (see lnk_cv_classify_task) + LNK_CvClassifyTask classify = {0}; + classify.input = &input; + classify.debug_p_arr = debug_p_arr; + classify.rrt_hm = &rrt_hm; + classify.obj_arr = obj_arr; + classify.class_tag = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + classify.warn_multi = push_array(scratch.arena, U8, obj_count ? obj_count : 1); + tp_for_parallel_prof(tp, 0, obj_count, lnk_cv_classify_task, &classify, "Classify Objs (parallel)"); + + // serial obj-order compaction into the 3 ordered arrays + deterministic warning emission. + // Cache-linear single pass; preserves the exact element order + warning order of the old loop. + for EachIndex(obj_idx, obj_count) { + U32Array *arr_ptr; + switch (classify.class_tag[obj_idx]) { + case 0: arr_ptr = &input.debug_p_indices; break; + case 1: arr_ptr = &input.ext_obj_indices; break; + default: arr_ptr = &input.int_obj_indices; break; + } + arr_ptr->v[arr_ptr->count++] = obj_idx; - if (debug_t->count == 0 && debug_p->count > 0) { - *debug_t = *debug_p; - } else if (debug_t->count && debug_p->count) { - lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); - MemoryZeroStruct(debug_t); - MemoryZeroStruct(debug_p); + if (classify.warn_multi[obj_idx]) { + lnk_error_obj(LNK_Warning_MultipleDebugTAndDebugP, obj_arr[obj_idx], "multiple sections with debug types detected, obj must have either .debug$T or .debug$P; discarding both sections"); + } } } @@ -1091,6 +2106,12 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, } ProfEnd(); + // resolve MSVC header-unit IFC debug records (LF_IFC_RECORD 0x1522) -> real CodeView types. + // injects .ifc debug-records blobs as extra objs and registers placeholder-TI redirects. + if (config->ifc_debug_records == LNK_SwitchState_Yes && config->ifc_map_list.node_count) { + lnk_apply_ifc_debug_records(tp, tp_arena, &input, config); + } + // set default min type index for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { input.min_type_indices[ti_source] = CV_MinComplexTypeIndex; } @@ -1107,25 +2128,24 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, ProfBegin("Make Symbol Inputs"); { - // count symbol blocks - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - input.symbol_input_count += s.node_count; - } - + // count symbol blocks (cache each obj's Symbols sub-section list so the fill pass below + // does not re-decode .debug$S a second time -- cv_sub_section_from_debug_s walks subsections). + String8List *per_obj_syms = push_array(scratch.arena, String8List, input.count ? input.count : 1); + LNK_CvSymTask sym_task = {0}; + sym_task.input = &input; + sym_task.per_obj_syms = per_obj_syms; + sym_task.counts = push_array(scratch.arena, U64, input.count ? input.count : 1); + + // parallel: cache each obj's Symbols sub-section list + count nodes + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_count_task, &sym_task, "Count Symbol Inputs"); + input.symbol_input_count = sum_array_u64(input.count, sym_task.counts); + sym_task.offsets = offsets_from_counts_array_u64(scratch.arena, sym_task.counts, input.count); + // alloc block pointers - input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count); + input.symbol_inputs = push_array_no_zero(tp_arena->v[0], LNK_SymbolInput, input.symbol_input_count ? input.symbol_input_count : 1); - U64 symbol_input_count = 0; - for EachIndex(obj_idx, input.count) { - String8List s = cv_sub_section_from_debug_s(input.debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols); - for EachNode(n, String8Node, s.first) { - Assert(symbol_input_count < input.symbol_input_count); - LNK_SymbolInput *in = &input.symbol_inputs[symbol_input_count++]; - in->obj_idx = obj_idx; - in->raw_symbols = n->string; - } - } + // parallel fill into disjoint per-obj ranges at prefix-sum offsets (byte-identical order) + tp_for_parallel_prof(tp, 0, input.count, lnk_cv_sym_fill_task, &sym_task, "Fill Symbol Inputs"); ProfBegin("Make Ranges"); @@ -1134,7 +2154,8 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, U64 max_weight = CeilIntegerDiv(total_input_size, tp->worker_count); U64 cursor = 0; - input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_ranges = push_array(tp_arena->v[0], Rng1U64, tp->worker_count); + input.symbol_input_range_count = tp->worker_count; for EachIndex(i, tp->worker_count) { if (cursor >= input.symbol_input_count) { break; } U64 begin = cursor; @@ -1185,6 +2206,25 @@ lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp_arena, LNK_Config *config, internal LNK_LeafRef lnk_leaf_ref_from_ti(LNK_CodeViewInput *input, U32 obj_idx, CV_TypeIndexSource source, CV_TypeIndex ti) { + // IFC redirect: a consuming obj's local LF_IFC_RECORD placeholder TI is mapped + // to a leaf inside an injected .ifc debug-records blob obj. The blob leaves then + // dedup/hash/fixup natively through the rest of this function. + if (input->has_ifc_redirects && source == CV_TypeIndexSource_TPI) { + // exact per-obj bitset filter: bit set iff Compose64Bit(obj_idx, ti) is a key in + // ifc_redirect_hm. skips the (miss-dominated) per-call key hash + map walk; on a set + // bit the original map search runs unchanged, so behavior is bit-identical. + U64 *bits = input->ifc_redirect_bits[obj_idx]; + if (bits != 0 && contains_1u64(input->ifc_redirect_ti_rng[obj_idx], ti)) { + U64 rel = ti - input->ifc_redirect_ti_rng[obj_idx].min; + if (bits[rel >> 6] & (1ull << (rel & 63))) { + U64 *packed = hash_map_search_u64_u64(&input->ifc_redirect_hm, Compose64Bit(obj_idx, ti)); + if (packed) { + return (LNK_LeafRef){ (U32)(*packed >> 32), (U32)(*packed & max_U32) }; + } + } + } + } + // ti range: external type server U64 ts_idx = input->obj_to_ts[obj_idx]; if (ts_idx != max_U64) { @@ -1260,12 +2300,13 @@ lnk_match_leaf_ref(LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b) } internal U64 -lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles) +lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles) { CV_DebugT *debug_t = &input->debug_t_arr[leaf_ref.obj_idx]; CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_ref.leaf_idx); CV_TypeIndexSource curr_ti_source = cv_type_index_source_from_leaf_kind(leaf.kind); CV_TypeIndex curr_ti = cv_ti_from_leaf_idx(debug_t, curr_ti_source, leaf_ref.leaf_idx); + U64 ti_count = cv_ti_offsets_count(&ti_offs); // init hasher LNK_Hasher hasher; @@ -1274,11 +2315,12 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // hash bytes around indices { U64 last_ti_off = 0; - for EachNode(ti_info, CV_TypeIndexInfo, ti_info_list.first) { + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); U8 *bytes = leaf.data.str + last_ti_off; - U64 size = ti_info->offset - last_ti_off; + U64 size = ti_info.offset - last_ti_off; lnk_hasher_update(&hasher, bytes, size); - last_ti_off = ti_info->offset + sizeof(CV_TypeIndex); + last_ti_off = ti_info.offset + sizeof(CV_TypeIndex); } Assert(leaf.data.size >= last_ti_off); @@ -1288,17 +2330,18 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf } // mix-in sub leaf hashes - for EachNode(sub_ti_n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n->offset, sizeof(*sub_ti_ptr)); + for (U64 ti_idx = 0; ti_idx < ti_count; ti_idx += 1) { + CV_TiOff sub_ti_n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *sub_ti_ptr = str8_deserial_get_raw_ptr(leaf.data, sub_ti_n.offset, sizeof(*sub_ti_ptr)); CV_TypeIndex sub_ti = memory_read32(sub_ti_ptr); - - // simple indices are stable across compile units - if (sub_ti < debug_t->ti_ranges[sub_ti_n->source].min) { + + // simple indices are stable across compile units + if (sub_ti < debug_t->ti_ranges[sub_ti_n.source].min) { lnk_hasher_update_struct(&hasher, &sub_ti); continue; } - if (sub_ti >= debug_t->ti_ranges[sub_ti_n->source].max) { + if (sub_ti >= debug_t->ti_ranges[sub_ti_n.source].max) { // discard type U32 leaf_idx = curr_ti - debug_t->ti_ranges[curr_ti_source].min; U8 *leaf_header = debug_t->data.str + debug_t->offsets[leaf_idx]; @@ -1311,7 +2354,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) out of bounds type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[leaf_ref.obj_idx], "%S", error_msg); scratch_end(scratch); @@ -1333,7 +2376,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf // log error Temp scratch = scratch_begin(0,0); String8 leaf_kind_str = cv_string_from_leaf_kind(leaf.kind); - String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, sub_ti_n->offset); + String8 error_msg = push_str8f(scratch.arena, "LF_%S(type_index: 0x%x) forward refs member type index 0x%x (leaf struct offset: 0x%llx)", leaf_kind_str, curr_ti, sub_ti, (U64)sub_ti_n.offset); lnk_error_obj(LNK_Error_InvalidTypeIndex, input->obj_arr[leaf_ref.obj_idx], "%S", error_msg); scratch_end(scratch); @@ -1341,7 +2384,7 @@ lnk_hash_cv_leaf(LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInf } // type index -> hash - LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, leaf_ref.obj_idx, sub_ti_n->source, sub_ti); + LNK_LeafRef sub_ref = lnk_leaf_ref_from_ti(input, leaf_ref.obj_idx, sub_ti_n.source, sub_ti); U64 sub_hash = input->debug_h_arr[sub_ref.obj_idx].v[sub_ref.leaf_idx]; // mix-in sub-type hash @@ -1367,15 +2410,16 @@ internal void lnk_hash_cv_leaf_deep(Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef root_leaf_ref, - CV_TypeIndexInfoList root_ti_info_list) + CV_TiOffsets root_ti_offs) { Temp temp = temp_begin(arena); typedef struct HashStack { struct HashStack *next; LNK_LeafRef leaf_ref; - CV_TypeIndexInfoList ti_info_list; - CV_TypeIndexInfo *ti_info; + CV_TiOffsets ti_offs; + U64 ti_next; + U64 ti_count; CV_Leaf leaf; CV_TypeIndex ti; CV_TypeIndexSource ti_source; @@ -1385,29 +2429,30 @@ lnk_hash_cv_leaf_deep(Arena *arena, CV_DebugT *root_debug_t = &input->debug_t_arr[root_leaf_ref.obj_idx]; HashStack *root_frame = push_array(temp.arena, HashStack, 1); root_frame->leaf_ref = root_leaf_ref; - root_frame->ti_info_list = root_ti_info_list; - root_frame->ti_info = root_ti_info_list.first; + root_frame->ti_offs = root_ti_offs; + root_frame->ti_next = 0; + root_frame->ti_count = cv_ti_offsets_count(&root_ti_offs); root_frame->leaf = cv_debug_t_get_leaf(root_debug_t, root_leaf_ref.leaf_idx); root_frame->ti_source = cv_type_index_source_from_leaf_kind(root_frame->leaf.kind); root_frame->ti = cv_ti_from_leaf_idx(root_debug_t, root_frame->ti_source, root_leaf_ref.leaf_idx); HashStack *stack = root_frame; while (stack) { - while (stack->ti_info) { - CV_TypeIndexInfo *ti_info = stack->ti_info; + while (stack->ti_next < stack->ti_count) { + CV_TiOff ti_info = cv_ti_offset_at(&stack->ti_offs, stack->ti_next); // advance iterator - stack->ti_info = stack->ti_info->next; + stack->ti_next += 1; // get type index info - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info->offset, sizeof(*ti_ptr)); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(stack->leaf.data, ti_info.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip out of bounds indices - if ( ! contains_1u64(input->debug_t_arr[root_leaf_ref.obj_idx].ti_ranges[ti_info->source], ti)) { continue; } + if ( ! contains_1u64(input->debug_t_arr[root_leaf_ref.obj_idx].ti_ranges[ti_info.source], ti)) { continue; } // skip hashed types - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_leaf_ref.obj_idx, ti_info->source, ti); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(input, root_leaf_ref.obj_idx, ti_info.source, ti); if (input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] != 0) { continue; } input->debug_h_arr[leaf_ref.obj_idx].v[leaf_ref.leaf_idx] = 1; @@ -1415,17 +2460,18 @@ lnk_hash_cv_leaf_deep(Arena *arena, HashStack *frame = push_array(temp.arena, HashStack, 1); frame->leaf_ref = leaf_ref; frame->leaf = cv_debug_t_get_leaf(&input->debug_t_arr[leaf_ref.obj_idx], leaf_ref.leaf_idx); - frame->ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); - frame->ti_info = frame->ti_info_list.first; + frame->ti_offs = cv_leaf_ti_offsets(temp.arena, frame->leaf.kind, frame->leaf.data); + frame->ti_next = 0; + frame->ti_count = cv_ti_offsets_count(&frame->ti_offs); frame->ti = ti; - frame->ti_source = ti_info->source; + frame->ti_source = ti_info.source; SLLStackPush(stack, frame); break; } // no more type indices, pop frame - if ( ! stack->ti_info) { - lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_info_list, 0); + if (stack->ti_next >= stack->ti_count) { + lnk_hash_cv_leaf(input, stack->leaf_ref, stack->ti_offs, 0); SLLStackPop(stack); } } @@ -1458,10 +2504,10 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_task) U32 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; for EachIndex(leaf_idx, debug_t->count) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf(task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_list, 1); + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf(task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_offs, 1); temp_end(temp); } ProfEnd(); @@ -1474,17 +2520,64 @@ THREAD_POOL_TASK_FUNC(lnk_hash_debug_t_deep_task) LNK_MergeTypes *task = raw_task; U64 obj_idx = task->indices.v[task_id]; CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); for EachIndex(leaf_idx, debug_t->count) { if (task->input->debug_h_arr[obj_idx].v[leaf_idx] != 0) { continue; } - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); - CV_TypeIndexInfoList ti_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_hash_cv_leaf_deep(temp.arena, task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_list); + if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } + Temp temp = temp_begin(task->fixed_arenas[worker_id]); + CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_hash_cv_leaf_deep(temp.arena, task->input, (LNK_LeafRef){ obj_idx, leaf_idx }, ti_offs); temp_end(temp); } ProfEnd(); } +// Deterministic sampling for the unique-leaf estimator: process every K-th leaf POSITION per obj. +// Position-based (leaf_idx % K), never value-based, so the sampled set -- and therefore the +// estimate and the table caps -- is a pure function of the input, schedule-independent. The +// dominant duplication pattern is whole-stream duplication (the same PCH/type-server leaf sequence +// repeated across objs), where a unique hash sits at the SAME position in every copy: the sampled +// distinct count then scales ~1/K, which LNK_ESTIMATE_SAMPLE_SCALE compensates for. The scale is +// calibrated (see the estimate block in lnk_merge_types); an undershoot is caught by the existing +// deterministic overflow-retry at total-based caps, an overshoot is clamped by Min(fallback cap). +// +// SCALE calibration: sampled-distinct is between distinct (fully position-scattered duplication) +// and distinct/K (whole-stream duplication or unique-heavy input), so the true ratio is in [1, K]. +// SCALE * 1.9 (the downstream safety factor) must cover the worst-case ratio K to keep the +// overflow-retry off for every duplication pattern: SCALE = 5.0 gives 5.0*1.9 = 9.5 >= K = 8 +// (1.19x margin over the bound, which also absorbs linear-counting noise). Measured on the FN +// editor-scale link: ratio 3.99 (TPI) / 6.13 (IPI); SCALE = 5.0 reproduces the unsampled +// estimator's caps exactly (64M/16M) at load factors 0.35/0.39. +#define LNK_ESTIMATE_SAMPLE_STRIDE 8 +#define LNK_ESTIMATE_SAMPLE_SCALE 5.0 + +internal +THREAD_POOL_TASK_FUNC(lnk_estimate_unique_leaves_task) +{ + ProfBeginFunction(); + LNK_MergeTypes *task = raw_task; + U64 obj_idx = task->indices.v[task_id]; + CV_DebugT *debug_t = &task->input->debug_t_arr[obj_idx]; + CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; + // same prune rule as lnk_leaf_dedup_task: NOTYPE'd IFC blob leaves were never hashed and are + // never inserted, so they must not contribute to the estimate either + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); + for (U64 leaf_idx = 0; leaf_idx < debug_t->count; leaf_idx += LNK_ESTIMATE_SAMPLE_STRIDE) { + CV_LeafHeader *header = cv_debug_t_get_leaf_header(debug_t, leaf_idx); + CV_LeafKind kind = memory_read16(MemberFromPtr(CV_LeafHeader, header, kind)); + if (is_ifc_blob && kind == CV_LeafKind_NOTYPE) { continue; } + CV_TypeIndexSource leaf_source = cv_type_index_source_from_leaf_kind(kind); + U64 bit_idx = debug_h->v[leaf_idx] & (task->estimate_bitmap_bits[leaf_source] - 1); + U32 *word = &task->estimate_bitmap[leaf_source][bit_idx / 32]; + U32 bit = 1u << (bit_idx % 32); + // atomic OR is commutative -> final bitmap contents are schedule-independent (deterministic); + // pre-check skips the interlocked op for already-set bits (the common case on dup-heavy input) + if ((ins_atomic_u32_eval(word) & bit) == 0) { ins_atomic_u32_or(word, bit); } + } + ProfEnd(); +} + internal THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) { @@ -1495,6 +2588,10 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) CV_DebugH *debug_h = &task->input->debug_h_arr[task->pop_obj_idx]; for EachInRange(leaf_idx, task->pop_range[task_id]) { + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } + LNK_LeafRef *bucket = 0; // alloc new bucket and assign type ref @@ -1537,7 +2634,13 @@ THREAD_POOL_TASK_FUNC(lnk_populate_leaf_ht) } while (idx != best_idx); is_inserted_or_updated = 0; exit:; - Assert(is_inserted_or_updated); + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } } @@ -1550,8 +2653,14 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) CV_DebugH *debug_h = &task->input->debug_h_arr[obj_idx]; ProfBeginDynamic("dedup in obj 0x%llx (%.*s) leaf count %llu", obj_idx, str8_varg(task->input->obj_arr[obj_idx]->path), debug_t->count); + B32 is_ifc_blob = task->input->has_ifc_redirects && contains_1u64(task->input->ifc_obj_range, obj_idx); + LNK_LeafRef *bucket = 0; for EachIndex(leaf_idx, debug_t->count) { + // another worker overflowed an estimate-sized table -- the whole dedup result is discarded + // and retried with the total-based caps, so bail out early + if (ins_atomic_u32_eval(&task->leaf_ht_overflow) != 0) { break; } + if (is_ifc_blob && cv_debug_t_get_leaf_header(debug_t, leaf_idx)->kind == CV_LeafKind_NOTYPE) { continue; } // alloc new bucket and assign type ref if (bucket == 0) { bucket = push_array_no_zero(arena, LNK_LeafRef, 1); } @@ -1592,7 +2701,13 @@ THREAD_POOL_TASK_FUNC(lnk_leaf_dedup_task) } while (idx != best_idx); is_inserted_or_updated = 0; exit:; - Assert(is_inserted_or_updated); + if (!is_inserted_or_updated && leaf_source != CV_TypeIndexSource_NULL) { + // TPI/IPI table is full (estimate undershot) -- flag for a deterministic retry with the + // total-based caps. the NULL-source table is deliberately undersized and silently drops + // leaves that do not fit (pre-existing behavior; they are never emitted). + ins_atomic_u32_eval_assign(&task->leaf_ht_overflow, 1); + break; + } } ProfEnd(); @@ -1829,17 +2944,18 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) } internal void -lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) +lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TiOffsets ti_offs) { - for EachNode(n, CV_TypeIndexInfo, ti_info_list.first) { - CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n->offset, sizeof(*ti_ptr)); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + CV_TypeIndex *ti_ptr = str8_deserial_get_raw_ptr(data, n.offset, sizeof(*ti_ptr)); CV_TypeIndex ti = memory_read32(ti_ptr); // skip basic types - if (ti < ctx->input->min_type_indices[n->source]) { continue; } + if (ti < ctx->input->min_type_indices[n.source]) { continue; } - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n->source], ctx->input, leaf_ref); + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n.source, ti); + CV_TypeIndex final_ti = lnk_assigned_ti_hash_search(&ctx->assigned_ti_arr[n.source], ctx->input, leaf_ref); memory_write32(ti_ptr, final_ti); #if LNK_PARANOID @@ -1859,15 +2975,11 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_symbols_task) for EachInRange(i, range) { LNK_SymbolInput symbols = task->input->symbol_inputs[i]; for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - Temp temp = temp_begin(task->fixed_arenas[worker_id]); - CV_Symbol symbol = {0}; TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - CV_TypeIndexInfoList ti_info_list = cv_get_symbol_type_index_offsets(temp.arena, symbol.kind, symbol.data); - lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_info_list); - - temp_end(temp); + CV_TiOffsets ti_offs = cv_symbol_ti_offsets(symbol.kind, symbol.data); + lnk_fixup_cv_type_indices(task, symbols.obj_idx, symbol.data, ti_offs); } } ProfEnd(); @@ -1883,45 +2995,62 @@ THREAD_POOL_TASK_FUNC(lnk_cv_patcher_inlines_task) Arena *fixed_arena = task->fixed_arenas[worker_id]; for EachNode(inline_data_n, String8Node, inlinee_lines.first) { Temp temp = temp_begin(fixed_arena); - CV_TypeIndexInfoList ti_info_list = cv_get_inlinee_type_index_offsets(temp.arena, inline_data_n->string); - lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_info_list); + CV_TiOffsets ti_offs = cv_inlinee_ti_offsets(temp.arena, inline_data_n->string); + lnk_fixup_cv_type_indices(task, obj_idx, inline_data_n->string, ti_offs); temp_end(temp); } ProfEnd(); } internal -THREAD_POOL_TASK_FUNC(lnk_cv_patcher_leaves_task) +THREAD_POOL_TASK_FUNC(lnk_count_unique_leaf_sizes_task) +{ + LNK_MergeTypes *task = raw_task; + Rng1U64 range = task->ranges[task_id]; + U64 size = 0; + for EachInRange(i, range) { + LNK_LeafRef *leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + CV_DebugT *debug_t = &task->input->debug_t_arr[leaf_ref->obj_idx]; + size += cv_debug_t_get_raw_leaf(debug_t, leaf_ref->leaf_idx).size; + } + task->leaf_buffer_offsets[task_id] = size; // exclusive-scanned into offsets on the main thread +} + +// Materialize unique leaves: copy each unique raw leaf (existing sorted order) into one contiguous +// private buffer and apply the type-index fixup to the COPY. This fuses the old +// lnk_cv_patcher_leaves_task (which patched TIs in-place into the mapped input, dirtying one +// copy-on-write page per touched .debug$T page) with the old lnk_unbucket_raw_leaves_task (which +// pointed result.v into the input). result.v now points into the copy: identical bytes, identical +// order, clean input pages. +internal +THREAD_POOL_TASK_FUNC(lnk_materialize_unique_leaves_task) { ProfBeginFunction(); LNK_MergeTypes *task = raw_task; Rng1U64 range = task->ranges[task_id]; Arena *fixed_arena = task->fixed_arenas[task_id]; - for EachInRange(leaf_ref_idx, range) { + U8 *cursor = task->leaf_buffer + task->leaf_buffer_offsets[task_id]; + for EachInRange(i, range) { + LNK_LeafRef *leaf_ref = task->unique_leaf_refs_arr[task->ti_source].v[i]; + CV_DebugT *debug_t = &task->input->debug_t_arr[leaf_ref->obj_idx]; + String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_ref->leaf_idx); + + // copy raw leaf into the private buffer + MemoryCopy(cursor, raw_leaf.str, raw_leaf.size); + task->result.v[task->ti_source][i] = cursor; + cursor += raw_leaf.size; + + // fixup type indices on the copy (same math the in-place leaf patcher applied) Temp temp = temp_begin(fixed_arena); - LNK_LeafRef *patch = task->unique_leaf_refs_arr[task->ti_source].v[leaf_ref_idx]; - CV_DebugT *debug_t = &task->input->debug_t_arr[patch->obj_idx]; - CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, patch->leaf_idx); - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf.kind, leaf.data); - lnk_fixup_cv_type_indices(task, patch->obj_idx, leaf.data, ti_info_list); + CV_Leaf leaf = {0}; + cv_read_leaf(str8(task->result.v[task->ti_source][i], raw_leaf.size), 0, 1, &leaf); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_fixup_cv_type_indices(task, leaf_ref->obj_idx, leaf.data, ti_offs); temp_end(temp); } ProfEnd(); } -internal -THREAD_POOL_TASK_FUNC(lnk_unbucket_raw_leaves_task) -{ - LNK_MergeTypes *task = raw_task; - Rng1U64 range = task->ranges[task_id]; - for EachInRange(i, range) { - LNK_LeafRef leaf_ref = *task->unique_leaf_refs_arr[task->ti_source].v[i]; - CV_DebugT *debug_t = &task->input->debug_t_arr[leaf_ref.obj_idx]; - String8 raw_leaf = cv_debug_t_get_raw_leaf(debug_t, leaf_ref.leaf_idx); - task->result.v[task->ti_source][i] = raw_leaf.str; - } -} - internal THREAD_POOL_TASK_FUNC(lnk_unbucket_hashes_task) { @@ -2025,7 +3154,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK Temp scratch = temp_begin(lnk_get_huge_arena()); LNK_MergeTypes task = { .input = input }; - U64 max_ti_list_size = sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); + // scratch bound: CV_TiOff arrays for member-walk leaves are built with doubling growth + // (sum of caps <= ~4x entry count, entries <= max_U16/8 per leaf => <= ~512KB), plus + // deep-hash stack frames; 2x the legacy per-node list bound keeps comfortable headroom + U64 max_ti_list_size = 2 * sizeof(CV_TypeIndexInfo) * (max_U16 / sizeof(CV_TypeIndex)); task.fixed_arenas = alloc_fixed_size_arena_array(scratch.arena, tp->worker_count, max_ti_list_size, max_ti_list_size); ProfBegin("Produce Hashes"); @@ -2036,6 +3168,7 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK U32Array indices; U32Array hash_indices; } hash_targets[] = { + { lnk_hash_debug_t_deep_task, input->ifc_indices }, // hash .ifc blobs first: int-obj leaves redirect into them { lnk_hash_debug_t_task, input->debug_p_indices }, // hash .debug$P first so we can mix in hashes for precompiled sub leaves when hashing leaves in .debug$T { lnk_hash_debug_t_task, input->int_obj_indices }, { lnk_hash_debug_t_deep_task, input->type_server_indices }, @@ -2066,6 +3199,34 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // batch-populate the .debug$T/$P leaf data the hashers below walk leaf by + // leaf; under farm-wide memory pressure these mapped pages were trimmed + // since the parse phase touched them (see lnk_prefetch_ranges) + ProfScope("Prefetch Type Data") + { + Temp temp = temp_begin(scratch.arena); + + U64 range_cap = 0; + for EachElement(i, hash_targets) { range_cap += hash_targets[i].hash_indices.count; } + + Rng1U64 *ranges = push_array_no_zero(temp.arena, Rng1U64, range_cap); + U64 range_count = 0; + for EachElement(i, hash_targets) { + for EachIndex(k, hash_targets[i].hash_indices.count) { + String8 data = input->debug_t_arr[hash_targets[i].hash_indices.v[k]].data; + if (data.size) { ranges[range_count++] = rng_1u64((U64)data.str, (U64)data.str + data.size); } + } + } + U64 prefetch_begin_us = now_time_us(); + U64 prefetch_bytes = 0; + for EachIndex(range_idx, range_count) { prefetch_bytes += dim_1u64(ranges[range_idx]); } + lnk_prefetch_ranges(tp, range_count, ranges); + lnk_log(LNK_Log_Timers, "[merge] prefetched %llu type data ranges (%llu MiB) in %.2f ms", + range_count, prefetch_bytes / MB(1), (F64)(now_time_us() - prefetch_begin_us) / 1000.0); + + temp_end(temp); + } + for EachElement(i, hash_targets) { task.indices = hash_targets[i].hash_indices; ProfBegin("Hash [Count: %.*s]", str8_varg(str8_from_count(scratch.arena, task.indices.count))); @@ -2092,26 +3253,109 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } ProfEnd(); + // bucket_arr (the ~1.3x-total-leaf-count probe tables) is only live through the dedup + extract + // phases: its last read is in lnk_get_present_buckets_task ("Copy present buckets") which copies + // bucket pointers into unique_leaf_refs. Allocate it in a dedicated arena so we can release that + // multi-GB working set immediately after the extract loop, before the merge-types/PDB-build peak. + // (A temp_begin on scratch.arena would not work: many surviving allocations -- unique_leaf_refs, + // assigned_ti, radix scratch -- land in scratch.arena after bucket_arr.) + Arena *bucket_arena = arena_alloc(.name = "LEAF_BUCKETS"); + ProfBegin("Leaf Hash Table Init"); - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - U64 total_count = 0; - for EachIndex(obj_idx, input->count) { total_count += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // fallback caps derived from TOTAL (pre-dedup, pre-prune) leaf counts. total >= unique always, so + // these caps can never overflow; they are also the caps the estimate-based sizing clamps against + // and retries with. NOTE: the NULL-source cap is deliberately derived from the pre-prune + // source_counts (see the IFC pruning comment in lnk_leaf_dedup_task) -- pruned NOTYPE leaves are + // never inserted, so pre-prune totals always cover the insert set with slack. + U64 leaf_ht_cap_fallback[CV_TypeIndexSource_COUNT] = {0}; + { + U64 total_counts[CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + for EachIndex(obj_idx, input->count) { total_counts[ti_source] += input->debug_t_arr[obj_idx].source_counts[ti_source]; } + // pow2 cap so bucket index is hash & (cap-1) (mask) instead of hash % cap (a 64-bit DIV in the + // densest dedup probe loop). u64_up_to_pow2(1.3*count) keeps load factor <= ~0.65. + leaf_ht_cap_fallback[ti_source] = u64_up_to_pow2(1 + ((total_counts[ti_source] * 13) / 10)); // * 1.3, pow2 + } + + // On dup-heavy input (PCH/type-server fan-out) unique count is a small fraction of total, and a + // total-sized probe table wastes multi-GB of demand-zero page faults on 64B-apart random probes. + // Estimate the distinct-hash count from the already-produced debug_h hashes (Produce Hashes + // completes above) with a per-source presence bitmap + linear counting, and size the tables from + // that instead. Everything here is a pure function of the input hashes, so the caps -- and the + // overflow/retry decision below -- are identical run to run. + ProfBegin("Estimate Unique Leaves"); + U64 estimate_begin_us = now_time_us(); + { + // sweep exactly the objs whose leaves get inserted: prepopulate + the four dedup passes + U32Array sweep_arrs[] = { input->debug_p_indices, input->int_obj_indices, input->type_server_indices, input->ifc_indices }; + U32Array sweep_indices = {0}; + for EachElement(i, sweep_arrs) { sweep_indices.count += sweep_arrs[i].count; } + sweep_indices.v = push_array_no_zero(scratch.arena, U32, sweep_indices.count); + sweep_indices.count = 0; + for EachElement(i, sweep_arrs) { + MemoryCopy(sweep_indices.v + sweep_indices.count, sweep_arrs[i].v, sizeof(U32) * sweep_arrs[i].count); + sweep_indices.count += sweep_arrs[i].count; + } + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + // at most ceil(total/K) hashes are inserted under K-th-position sampling, so bits >= + // total/K >= sampled-unique keeps the bitmap load < 1, where linear counting is accurate; + // clamp keeps the transient bitmap allocation bounded (16MB per source at the top end) + U64 sampled_total = (total_counts[ti_source] + LNK_ESTIMATE_SAMPLE_STRIDE - 1) / LNK_ESTIMATE_SAMPLE_STRIDE; + task.estimate_bitmap_bits[ti_source] = u64_up_to_pow2(Clamp(1ull << 16, sampled_total, 1ull << 27)); + task.estimate_bitmap [ti_source] = push_array(scratch.arena, U32, task.estimate_bitmap_bits[ti_source] / 32); + } + + task.indices = sweep_indices; + tp_for_parallel(tp, 0, task.indices.count, lnk_estimate_unique_leaves_task, &task); + + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + U64 bit_count = task.estimate_bitmap_bits[ti_source]; + U64 word_count = bit_count / 32; + U64 set_count = 0; + for EachIndex(word_idx, word_count) { set_count += count_bits_set32(task.estimate_bitmap[ti_source][word_idx]); } + + U64 cap = leaf_ht_cap_fallback[ti_source]; + U64 zero_count = bit_count - set_count; + // the NULL-source table keeps the historic total-based cap: objs synthesized outside the + // parse path never populate source_counts, so its cap can be far below the number of + // NULL-source leaves thrown at it. that has always been tolerated -- NULL-source leaves are + // never emitted, the table just drops what does not fit (see the fall-through handling in + // lnk_leaf_dedup_task) -- so it must not participate in estimate sizing or overflow retry. + if (ti_source != CV_TypeIndexSource_NULL && set_count > 0 && zero_count > 0) { + // linear counting: distinct ~= m * ln(m / zeros) + F64 estimate = (F64)bit_count * log((F64)bit_count / (F64)zero_count); + if (estimate < (F64)set_count) { estimate = (F64)set_count; } + // scale the sampled distinct count back up to a full-population estimate (see the + // sampling comment above lnk_estimate_unique_leaves_task) + estimate *= LNK_ESTIMATE_SAMPLE_SCALE; + // 1.9x safety keeps the load factor <= ~0.55 even before pow2 rounding; overflow (only + // possible if the estimate undershoots by >1.8x) is caught and retried deterministically + U64 target = (U64)(estimate * 1.9) + 4096; + cap = Min(u64_up_to_pow2(target), leaf_ht_cap_fallback[ti_source]); + lnk_log(LNK_Log_Timers, "[typededup] estimate src=%llu: set=%llu est=%.0f cap=%llu (fallback %llu)", + ti_source, set_count, estimate, cap, leaf_ht_cap_fallback[ti_source]); + } + task.leaf_ht_arr[ti_source].cap = cap; + } + } + lnk_log(LNK_Log_Timers, "[typededup] unique-leaf estimate in %.2f ms", (F64)(now_time_us() - estimate_begin_us) / 1000.0); + ProfEnd(); - task.leaf_ht_arr[ti_source].cap = total_count; - task.leaf_ht_arr[ti_source].cap = 1 + ((task.leaf_ht_arr[ti_source].cap * 13) / 10); // * 1.3 - task.leaf_ht_arr[ti_source].bucket_arr = push_array(scratch.arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].bucket_arr = push_array(bucket_arena, LNK_LeafRef *, task.leaf_ht_arr[ti_source].cap); #if PROFILE_TELEMETRY - tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); + tmMessage(0, TMMF_ICON_NOTE, "%.*s Bucket Count: %.*s", str8_varg(cv_string_from_type_index_source(ti_source)), str8_varg(str8_from_count(scratch.arena, task.leaf_ht_arr[ti_source].cap))); #endif + } } ProfEnd(); U32Array dedup_type_server_indices = input->type_server_indices; - ProfBegin("Prepopulate hash table with largest type-set"); + LNK_TypeServer *largest_ts = 0; { - LNK_TypeServer *largest_ts = 0; for EachIndex(i, input->ts_arr.count) { LNK_TypeServer *ts = &input->ts_arr.v[i]; if (ts->rrt == 0) { continue; } @@ -2123,7 +3367,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK if (largest_ts) { task.pop_obj_idx = input->ts_obj_range.min + largest_ts->ts_idx; task.pop_range = tp_divide_work(scratch.arena, task.input->debug_t_arr[task.pop_obj_idx].count, tp->worker_count); - tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); U32Array new_dedup_type_server_indices = { .v = push_array(scratch.arena, U32, input->type_server_indices.count) }; for EachIndex(i, input->type_server_indices.count) { @@ -2133,18 +3376,42 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK dedup_type_server_indices = new_dedup_type_server_indices; } } - ProfEnd(); - ProfBegin("Leaf Dedup"); - task.indices = input->debug_p_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); + for (U64 attempt = 0; ; attempt += 1) { + ProfBegin("Prepopulate hash table with largest type-set"); + if (largest_ts) { + tp_for_parallel(tp, tp_temp, tp->worker_count, lnk_populate_leaf_ht, &task); + } + ProfEnd(); - task.indices = input->int_obj_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); + ProfBegin("Leaf Dedup"); + task.indices = input->debug_p_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$P"); - task.indices = dedup_type_server_indices; - tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); - ProfEnd(); + task.indices = input->int_obj_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, ".debug$T"); + + task.indices = dedup_type_server_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "Type Servers"); + + task.indices = input->ifc_indices; + tp_for_parallel_prof(tp, tp_temp, task.indices.count, lnk_leaf_dedup_task, &task, "IFC Blobs"); + ProfEnd(); + + if (ins_atomic_u32_eval(&task.leaf_ht_overflow) == 0) { break; } + + // an estimate-sized table overflowed: rebuild every probe table at the always-sufficient + // total-based caps and redo the passes. the input hashes are deterministic, so this branch is + // taken (or not) identically every run, and the retried result is what the total-based sizing + // would have produced in the first place. the fallback caps cannot overflow, so at most one retry. + AssertAlways(attempt == 0); + lnk_log(LNK_Log_Debug, "leaf dedup: unique-count estimate overflowed, retrying with total-based table caps"); + ins_atomic_u32_eval_assign(&task.leaf_ht_overflow, 0); + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { + task.leaf_ht_arr[ti_source].cap = leaf_ht_cap_fallback[ti_source]; + task.leaf_ht_arr[ti_source].bucket_arr = push_array(bucket_arena, LNK_LeafRef *, leaf_ht_cap_fallback[ti_source]); + } + } ProfBegin("Extract present buckets from the leaf hash tables"); @@ -2159,6 +3426,10 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); + lnk_log(LNK_Log_Timers, "[typededup] src=%llu: unique=%llu cap=%llu load=%.3f", + ti_source, task.unique_leaf_refs_arr[ti_source].count, task.leaf_ht_arr[ti_source].cap, + task.leaf_ht_arr[ti_source].cap ? (F64)task.unique_leaf_refs_arr[ti_source].count / (F64)task.leaf_ht_arr[ti_source].cap : 0.0); + // sort output leaves based on { location index, leaf index } to guarantee determinism { LNK_LeafRefArray arr = task.unique_leaf_refs_arr[ti_source]; @@ -2242,6 +3513,16 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK } } + // bucket_arr is fully consumed (copied into unique_leaf_refs / sorted) -- release the probe tables + // now so this multi-GB working set is gone before the merge-types/PDB-build peak. Handed to a + // background reaper thread: a serial VirtualFree(MEM_RELEASE) of these multi-GB committed blocks + // costs ~350ms+ of main-thread kernel time (MiDeleteVaDirect/MiDecommitFreePage), which otherwise + // sits on the critical path between dedup and the type-index fixup passes. All pointers into the + // arena are dropped below before the launch returns ownership to the reaper. + for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.leaf_ht_arr[ti_source].bucket_arr = 0; } + if (g_arena_reaper_thread.u64[0] != 0) { thread_join(g_arena_reaper_thread, max_U64); } + g_arena_reaper_thread = thread_launch(lnk_arena_release_thread, bucket_arena); + #if PROFILE_TELEMETRY tmMessage(0, TMMF_ICON_NOTE, "TPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_TPI].count))); tmMessage(0, TMMF_ICON_NOTE, "IPI Count: %.*s", str8_varg(str8_from_count(scratch.arena, task.unique_leaf_refs_arr[CV_TypeIndexSource_IPI].count))); @@ -2257,10 +3538,11 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.assigned_ti_arr[ti_source].cap = ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); - // unique extraction is complete, so the dedup bucket slots can back the - // direct hash table without increasing peak memory - Assert(task.assigned_ti_arr[ti_source].cap <= task.leaf_ht_arr[ti_source].cap); - task.assigned_ti_arr[ti_source].hash_arr = (U64 *)task.leaf_ht_arr[ti_source].bucket_arr; + // bucket_arr used to back hash_arr here to avoid a fresh allocation, but the bucket + // arena is released right after the extract loop (above) so the probe tables' multi-GB + // working set is gone before the merge-types commit peak; allocate hash_arr fresh -- + // it is sized by the (much smaller) unique count, not the total-based bucket cap + task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); task.min_type_indices[ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); @@ -2276,11 +3558,9 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK tp_for_parallel_prof(tp, 0, input->count, lnk_cv_patcher_inlines_task, &task, "Fixup Inlines Type Indices"); } - for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { - task.ti_source = ti_source; - task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_cv_patcher_leaves_task, &task, "Fixup Types Type Indices"); - } + // NOTE: the leaf TI-fixup is fused into the unbucket/materialize pass below -- it copies each + // unique leaf into a private buffer and patches the copy, instead of patching the mapped input + // (which copy-on-writes one page per touched .debug$T page). } ProfEnd(); @@ -2307,7 +3587,21 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.result.count[ti_source] = unique_leaf_refs.count; task.result.v [ti_source] = push_array(tp_temp->v[0], U8 *, unique_leaf_refs.count); task.ranges = tp_divide_work(scratch.arena, unique_leaf_refs.count, tp->worker_count); - tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_unbucket_raw_leaves_task, &task, "Unbucket Leaves"); + + // per-lane byte totals for the materialize buffer, exclusive-scanned into offsets + task.leaf_buffer_offsets = push_array_no_zero(scratch.arena, U64, tp->worker_count + 1); + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_count_unique_leaf_sizes_task, &task, "Count Leaf Sizes"); + { + U64 acc = 0; + for EachIndex(lane, tp->worker_count) { + U64 lane_size = task.leaf_buffer_offsets[lane]; + task.leaf_buffer_offsets[lane] = acc; + acc += lane_size; + } + task.leaf_buffer_offsets[tp->worker_count] = acc; + task.leaf_buffer = push_array_no_zero(tp_temp->v[0], U8, acc ? acc : 1); + } + tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_materialize_unique_leaves_task, &task, "Materialize + Fixup Leaves"); if (merge_flags & LNK_MergeTypeFlag_ExportHashes) { task.result.hashes[ti_source] = push_array_no_zero(tp_temp->v[0], U64, unique_leaf_refs.count); @@ -2541,23 +3835,30 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) ProfBegin("Global Symbols"); { + // FAIR-SHARE: symbol_input_ranges is a FIXED [symbol_input_range_count] partition built at + // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count + // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is processed + // exactly once for any C; at C == fixed this degenerates to lane == task_id (one lane each, + // identical to the old direct indexing). VoidList global_symbols = {0}; - for EachInRange(i, task->cv->symbol_input_ranges[task_id]) { - LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; - for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { - CV_Symbol symbol = {0}; - TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + for (U64 lane = task_id; lane < task->cv->symbol_input_range_count; lane += tp->worker_count) { + for EachInRange(i, task->cv->symbol_input_ranges[lane]) { + LNK_SymbolInput symbols = task->cv->symbol_inputs[i]; + for (U64 cursor = 0, depth = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); - if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { - void *ptr = cv_ptr_from_symbol(symbol); - void_list_push(scratch.arena, &global_symbols, ptr); - } + if (cv_is_global_symbol(symbol.kind) || (depth == 0 && cv_is_typedef(symbol.kind))) { + void *ptr = cv_ptr_from_symbol(symbol); + void_list_push(scratch.arena, &global_symbols, ptr); + } - if (cv_is_scope_symbol(symbol.kind)) { - depth += 1; - } else if (cv_is_end_symbol(symbol.kind)) { - if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } - depth -= 1; + if (cv_is_scope_symbol(symbol.kind)) { + depth += 1; + } else if (cv_is_end_symbol(symbol.kind)) { + if (depth == 0) { Assert(0 && "malformed symbol stream"); break; } + depth -= 1; + } } } } @@ -2567,42 +3868,93 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) U64 bucket_cap; void **buckets; + U64 *collect_counts; // [worker_count] + void **flat_symbols; // [global_symbol_count] if (task_id == 0) { - bucket_cap = global_symbol_count * 13 / 10; - buckets = push_array(scratch.arena, void *, bucket_cap); + bucket_cap = global_symbol_count * 13 / 10; + buckets = push_array(scratch.arena, void *, bucket_cap); + collect_counts = push_array(scratch.arena, U64, tp->worker_count); + flat_symbols = push_array_no_zero(scratch.arena, void *, global_symbol_count ? global_symbol_count : 1); } tp_broadcast(&bucket_cap); tp_broadcast(&buckets); + tp_broadcast(&collect_counts); + tp_broadcast(&flat_symbols); + + // BALANCE: per-lane global-symbol density varies a lot (the collect partition above is + // weighted by raw symbol bytes, not by global-symbol count), which skewed the insert phase + // by ~2x. Flatten the per-worker lists (in worker order) into one array and re-divide the + // INSERTS evenly. The deduper is CAS-based and content-keyed: any insert partition yields + // the same deduped content set, and downstream order is already schedule-independent (the + // per-chain sort keys on the content hash written into n->data.offset below), so this only + // changes who performs an insert, never the output. + ProfBegin("Insert Global Symbols"); + collect_counts[task_id] = global_symbols.count; + barrier_wait(tp->barrier); + { + U64 flat_off = 0; + for (U64 w = 0; w < task_id; w += 1) { flat_off += collect_counts[w]; } + for EachNode(n, VoidNode, global_symbols.first) { flat_symbols[flat_off++] = n->v; } + } + barrier_wait(tp->barrier); - // insert symbols into hash table - for EachNode(n, VoidNode, global_symbols.first) { - String8 raw = cv_raw_from_symbol(n->v); - U64 hash = u64_hash_from_str8(raw); - cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, n->v); + // insert symbols into hash table (even split of the flattened array) + { + U64 ins_lo = (task_id * global_symbol_count) / tp->worker_count; + U64 ins_hi = ((task_id + 1) * global_symbol_count) / tp->worker_count; + for (U64 i = ins_lo; i < ins_hi; i += 1) { + String8 raw = cv_raw_from_symbol(flat_symbols[i]); + U64 hash = u64_hash_from_str8(raw); + cv_symbol_deduper_insert_or_update(buckets, bucket_cap, hash, flat_symbols[i]); + } } barrier_wait(tp->barrier); + ProfEnd(); - U64 symbol_count = 0; - void **symbol_arr = 0; // [symbol_count] - Rng1U64 *symbol_ranges = 0; // [worker_count] - U32 *symbol_hashes = 0; // [symbol_count] + // compact buckets in parallel: each worker owns a contiguous slot range, counts its + // occupied slots, then copies them to its prefix-sum offset. Concatenated ranges preserve + // ascending slot order, so symbol_arr is byte-identical to the old task-0-serial compaction + // (which stalled the other workers at the next barrier for the whole bucket_cap sweep). + U64 symbol_count = 0; + void **symbol_arr = 0; // [symbol_count] + Rng1U64 *symbol_ranges = 0; // [worker_count] + U32 *symbol_hashes = 0; // [symbol_count] + U64 *compact_counts = 0; // [worker_count] if (task_id == 0) { - ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); - for EachIndex(src, bucket_cap) { - buckets[symbol_count] = buckets[src]; - symbol_count += buckets[src] != 0; - } - ProfEnd(); + compact_counts = push_array(scratch.arena, U64, tp->worker_count); + } + tp_broadcast(&compact_counts); + + ProfBeginV("Compact Buckets [bucket_cap %llu]", bucket_cap); + U64 slot_lo = (task_id * bucket_cap) / tp->worker_count; + U64 slot_hi = ((task_id + 1) * bucket_cap) / tp->worker_count; + { + U64 c = 0; + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { c += buckets[slot] != 0; } + compact_counts[task_id] = c; + } + barrier_wait(tp->barrier); - symbol_arr = buckets; + for EachIndex(w, tp->worker_count) { symbol_count += compact_counts[w]; } // same value on every worker + if (task_id == 0) { + symbol_arr = push_array_no_zero(scratch.arena, void *, symbol_count ? symbol_count : 1); symbol_ranges = tp_divide_work(scratch.arena, symbol_count, tp->worker_count); symbol_hashes = push_array_no_zero(scratch.arena, U32, symbol_count); } - tp_broadcast(&symbol_count); tp_broadcast(&symbol_arr); tp_broadcast(&symbol_ranges); tp_broadcast(&symbol_hashes); + { + U64 dst = 0; + for (U64 w = 0; w < task_id; w += 1) { dst += compact_counts[w]; } + for (U64 slot = slot_lo; slot < slot_hi; slot += 1) { + if (buckets[slot]) { symbol_arr[dst++] = buckets[slot]; } + } + } + barrier_wait(tp->barrier); + ProfEnd(); + // hash symbols Rng1U64 symbol_range = symbol_ranges[task_id]; for EachInRange(i, symbol_range) { @@ -2612,17 +3964,40 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push global symbols + // push global symbols, sharded by bucket range: worker i owns buckets [i*B/W, (i+1)*B/W) and + // walks the FULL symbol sequence in global order, inserting only symbols whose bucket lands in + // its range. each bucket has a single owner and receives its inserts in global sequence order, + // so per-chain order (which is serialized into the PDB) is byte-identical to a serial loop, for + // any worker count -- no locks, no atomics. + CV_SymbolNode *global_nodes = 0; if (task_id == 0) { - CV_SymbolNode *nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); + global_nodes = push_array_no_zero(gsi->arena, CV_SymbolNode, symbol_count); + } + tp_broadcast(&global_nodes); + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; for EachIndex(i, symbol_count) { - CV_SymbolNode *n = &nodes[i]; + U64 bucket_idx = symbol_hashes[i] % gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + CV_SymbolNode *n = &global_nodes[i]; n->prev = n->next = 0; n->data = cv_symbol_from_ptr(symbol_arr[i]); - n->data.offset = i; - gsi_push_(gsi, symbol_hashes[i], n); + // deterministic same-name tie-break for the per-bucket sort in gsi_serialize_symbols_task + // (gsi_symbol_is_before compares name -> offset -> kind -> data bytes): key on the content + // hash of the full raw record instead of the compacted deduper slot index. Slot order is + // CAS-arrival order in cv_symbol_deduper_insert_or_update and can flip between same-name + // different-content records (e.g. duplicate S_UDTs with distinct type indices) whenever the + // lane->worker schedule changes (shared-pool cohorts) or probe chains contend, permuting + // symrec bytes run-to-run. The hash is schedule-independent; on collision the comparator's + // kind/data-bytes fallback stays content-deterministic, and byte-identical records cannot + // reach the sort (the deduper folds them), so the pointer tiebreaker stays unreachable. + n->data.offset = u64_hash_from_str8(cv_raw_from_symbol(symbol_arr[i])); + cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], n); } } + barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += symbol_count; } } ProfEnd(); @@ -2714,59 +4089,76 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) } barrier_wait(tp->barrier); - // push proc refs - if (task_id == 0) { - U64 total_proc_ref_count = sum_array_u64(tp->worker_count, proc_ref_counts); - for EachIndex(i, total_proc_ref_count) { gsi_push_(gsi, proc_ref_hashes[i], &proc_ref_nodes[i]); } + // push proc refs, sharded by bucket range (single owner per bucket, inserts in global node + // order -> per-chain order identical to a serial loop for any worker count) + { + U64 shard_min = (task_id * gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * gsi->bucket_count) / tp->worker_count; + for EachIndex(i, total_proc_ref_count) { + U64 bucket_idx = (U32)proc_ref_hashes[i] % gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + cv_symbol_list_push_node(&gsi->bucket_arr[bucket_idx], &proc_ref_nodes[i]); + } } barrier_wait(tp->barrier); + if (task_id == 0) { gsi->symbol_count += total_proc_ref_count; } } ProfEnd(); ProfBegin("Public Symbols"); { - U64 *public_symbol_sizes = 0; // [worker_count] - U64 *public_symbol_node_counts = 0; // [worker_count] + // FAIR-SHARE: task->symtab->chunks is a FIXED [symtab->arena->count] partition built at + // full pool width, but this barrier pass runs at the pinned cohort C == tp->worker_count + // (C <= fixed). Walk the fixed lanes strided by the cohort so every fixed lane is + // processed exactly once for any C, and keep every per-lane array in FIXED-lane order so + // the flattened global order below (which feeds the sharded PSI insert) is byte-identical + // to a full-width run. At C == fixed the strided loops degenerate to lane == task_id. + U64 fixed_lane_count = task->symtab->arena->count; + + U64 *public_symbol_sizes = 0; // [fixed_lane_count] + U64 *public_symbol_node_counts = 0; // [fixed_lane_count] if (task_id == 0) { - public_symbol_sizes = push_array(scratch.arena, U64, tp->worker_count); - public_symbol_node_counts = push_array(scratch.arena, U64, tp->worker_count); + public_symbol_sizes = push_array(scratch.arena, U64, fixed_lane_count); + public_symbol_node_counts = push_array(scratch.arena, U64, fixed_lane_count); } tp_broadcast(&public_symbol_sizes); tp_broadcast(&public_symbol_node_counts); // compute buffer size for CV public symbols - LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[task_id]; - U64 public_symbol_size = 0; - U64 public_symbol_count = 0; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - - public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); - public_symbol_count += 1; - public_symbol_node_counts[task_id] += 1; - } - } - public_symbol_sizes [task_id] += public_symbol_size; - public_symbol_node_counts[task_id] += public_symbol_count; + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + U64 public_symbol_size = 0; + U64 public_symbol_count = 0; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + public_symbol_size += AlignPow2(sizeof(CV_SymPub32) + symbol->name.size + 1, sizeof(void *)); + public_symbol_count += 1; + public_symbol_node_counts[lane] += 1; + } + } + public_symbol_sizes [lane] += public_symbol_size; + public_symbol_node_counts[lane] += public_symbol_count; + } barrier_wait(tp->barrier); Arena **public_symbol_arenas = 0; Arena **public_symbol_node_arenas = 0; - CV_SymbolList *public_symbols = 0; // [worker_count] - U32 **public_symbol_hashes = 0; // [worker_count][public_symbol.count] + CV_SymbolList *public_symbols = 0; // [fixed_lane_count] + U32 **public_symbol_hashes = 0; // [fixed_lane_count][public_symbol.count] if (task_id == 0) { - U64 public_symbol_total_count = sum_array_u64(tp->worker_count, public_symbol_node_counts); - public_symbol_arenas = alloc_arena_many(psi->gsi->arena, tp->worker_count, public_symbol_sizes); - public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, tp->worker_count, public_symbol_node_counts, CV_SymbolNode); - public_symbols = push_array(scratch.arena, CV_SymbolList, tp->worker_count); - public_symbol_hashes = push_array(scratch.arena, U32 *, tp->worker_count); + U64 public_symbol_total_count = sum_array_u64(fixed_lane_count, public_symbol_node_counts); + public_symbol_arenas = alloc_arena_many(psi->gsi->arena, fixed_lane_count, public_symbol_sizes); + public_symbol_node_arenas = alloc_arena_array(psi->gsi->arena, fixed_lane_count, public_symbol_node_counts, CV_SymbolNode); + public_symbols = push_array(scratch.arena, CV_SymbolList, fixed_lane_count); + public_symbol_hashes = push_array(scratch.arena, U32 *, fixed_lane_count); } tp_broadcast(&public_symbol_arenas); tp_broadcast(&public_symbol_node_arenas); @@ -2774,52 +4166,87 @@ THREAD_POOL_TASK_FUNC(lnk_move_global_symbols_to_gsi) tp_broadcast(&public_symbol_hashes); // make CV public symbols - Arena *public_symbol_arena = public_symbol_arenas [task_id]; - Arena *public_symbol_node_arena = public_symbol_node_arenas[task_id]; - CV_SymbolList *public_symbol_list = &public_symbols [task_id]; - for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { - for EachIndex(i, chunk->count) { - LNK_Symbol *symbol = chunk->v[i].symbol; - LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); - - // discard removed and non-section symbols - if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } - COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); - if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } - - CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; - ISectOff sc = lnk_sc_from_symbol(symbol); - CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); - cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + LNK_SymbolHashTrieChunkList symbol_chunks = task->symtab->chunks[lane]; + Arena *public_symbol_arena = public_symbol_arenas [lane]; + Arena *public_symbol_node_arena = public_symbol_node_arenas[lane]; + CV_SymbolList *public_symbol_list = &public_symbols [lane]; + for EachNode(chunk, LNK_SymbolHashTrieChunk, symbol_chunks.first) { + for EachIndex(i, chunk->count) { + LNK_Symbol *symbol = chunk->v[i].symbol; + LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + + // discard removed and non-section symbols + if (symbol_parsed.section_number == lnk_obj_get_removed_section_number(symbol_ref.obj)) { continue; } + COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); + if (symbol_interp != COFF_SymbolValueInterp_Regular) { continue; } + + CV_Pub32Flags flags = COFF_SymbolType_IsFunc(symbol_parsed.type) ? CV_Pub32Flag_Function : 0; + ISectOff sc = lnk_sc_from_symbol(symbol); + CV_Symbol pub_symbol = cv_make_pub32(public_symbol_arena, flags, safe_cast_u32(sc.off), safe_cast_u16(sc.isect), symbol->name); + cv_symbol_list_push(public_symbol_node_arena, public_symbol_list, pub_symbol); + } } } barrier_wait(tp->barrier); // hash public symbols - { + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { U64 hash_idx = 0; - U32 *hashes = push_array(scratch.arena, U32, public_symbols[task_id].count); - for EachNode(n, CV_SymbolNode, public_symbols[task_id].first) { + U32 *hashes = push_array(scratch.arena, U32, public_symbols[lane].count); + for EachNode(n, CV_SymbolNode, public_symbols[lane].first) { String8 name = cv_name_from_symbol(n->data.kind, n->data.data); hashes[hash_idx++] = gsi_hash(gsi, name); } - public_symbol_hashes[task_id] = hashes; + public_symbol_hashes[lane] = hashes; } barrier_wait(tp->barrier); - // insert public symbols into PSI + // flatten the per-worker symbol lists (in worker order, matching the old serial walk) into one + // global-order node/hash array, so the sharded insert below can walk it without racing on the + // list links + U64 public_symbol_total_count = 0; + U64 *public_symbol_offsets = 0; // [fixed_lane_count] + CV_SymbolNode **public_symbol_flat_nodes = 0; // [public_symbol_total_count] + U32 *public_symbol_flat_hashes = 0; // [public_symbol_total_count] if (task_id == 0) { - for EachIndex(i, tp->worker_count) { - U64 k = 0; - for (CV_SymbolNode *curr = public_symbols[i].first, *next = 0; curr != 0; curr = next, k += 1) { - next = curr->next; - curr->next = 0; - gsi_push_(psi->gsi, public_symbol_hashes[i][k], curr); - } + U64 *list_counts = push_array_no_zero(scratch.arena, U64, fixed_lane_count); + for EachIndex(i, fixed_lane_count) { list_counts[i] = public_symbols[i].count; } + public_symbol_offsets = offsets_from_counts_array_u64(scratch.arena, list_counts, fixed_lane_count); + public_symbol_total_count = sum_array_u64(fixed_lane_count, list_counts); + public_symbol_flat_nodes = push_array_no_zero(scratch.arena, CV_SymbolNode *, public_symbol_total_count); + public_symbol_flat_hashes = push_array_no_zero(scratch.arena, U32, public_symbol_total_count); + } + tp_broadcast(&public_symbol_total_count); + tp_broadcast(&public_symbol_offsets); + tp_broadcast(&public_symbol_flat_nodes); + tp_broadcast(&public_symbol_flat_hashes); + for (U64 lane = task_id; lane < fixed_lane_count; lane += tp->worker_count) { + U64 cursor = public_symbol_offsets[lane]; + U64 k = 0; + for (CV_SymbolNode *curr = public_symbols[lane].first; curr != 0; curr = curr->next, k += 1) { + public_symbol_flat_nodes [cursor] = curr; + public_symbol_flat_hashes[cursor] = public_symbol_hashes[lane][k]; + cursor += 1; + } + } + barrier_wait(tp->barrier); + + // insert public symbols into PSI, sharded by bucket range (single owner per bucket, inserts in + // global order -> per-chain order identical to a serial loop for any worker count) + { + PDB_GsiContext *pub_gsi = psi->gsi; + U64 shard_min = (task_id * pub_gsi->bucket_count) / tp->worker_count; + U64 shard_max = ((task_id + 1) * pub_gsi->bucket_count) / tp->worker_count; + for EachIndex(i, public_symbol_total_count) { + U64 bucket_idx = public_symbol_flat_hashes[i] % pub_gsi->bucket_count; + if (bucket_idx < shard_min || bucket_idx >= shard_max) { continue; } + cv_symbol_list_push_node(&pub_gsi->bucket_arr[bucket_idx], public_symbol_flat_nodes[i]); } } barrier_wait(tp->barrier); + if (task_id == 0) { psi->gsi->symbol_count += public_symbol_total_count; } } ProfEnd(); @@ -3264,6 +4691,391 @@ lnk_pdb_output_enqueue_remaining(LNK_PdbOutput *output, MSF_Context *msf) scratch_end(scratch); } +//////////////////////////////// +// Type Garbage Collection +// +// After type merging, prune merged TPI/IPI leaves that are not reachable from any surviving +// symbol record (the GC roots), compact them, and remap all type indices. link.exe keeps a +// large unreferenced-type set; pruning it is a transparent PDB-size win (debug-info only -- the +// image is untouched). Runs on the final post-fixup type indices in place. + +typedef struct LNK_GCTypes +{ + LNK_CodeViewInput *cv; + U64 min [CV_TypeIndexSource_COUNT]; // first type index per source + U64 orig_n[CV_TypeIndexSource_COUNT]; // pre-GC leaf count per source + U8 *mark [CV_TypeIndexSource_COUNT]; // reachable bitmap, indexed by (ti - min) + CV_TypeIndex *remap [CV_TypeIndexSource_COUNT]; // old leaf idx -> new type index + U8 **leaf_v [CV_TypeIndexSource_COUNT]; // original leaf pointer arrays + U32 *udt_next; // TPI fwdref<->definition unique_name ring + Rng1U64 *sym_ranges; + B32 do_rewrite; // 0 = mark roots, 1 = rewrite to compacted indices + // transitive-closure frontier: indices marked but not yet expanded. Each leaf is appended once + // (the atomic mark gates it), so frontier[s] is sized orig_n[s] and fcount[s] is its atomic tail. + U32 *frontier[CV_TypeIndexSource_COUNT]; + U32 *fcount [CV_TypeIndexSource_COUNT]; // atomic append cursor per source + // per-source scratch (set before dispatch) + CV_TypeIndexSource cur_source; + U8 **cur_leaf_v; + Rng1U64 *cur_ranges; + U64 round_begin, round_end; // frontier slice processed this round +} LNK_GCTypes; + +typedef struct LNK_GCNamePair { U64 hash; U32 idx; } LNK_GCNamePair; + +internal int +lnk_gc_name_pair_is_before(void *raw_a, void *raw_b) +{ + LNK_GCNamePair *a = raw_a, *b = raw_b; + return a->hash != b->hash ? (a->hash < b->hash) : (a->idx < b->idx); +} + +internal void +lnk_gc_mark_ti(LNK_GCTypes *g, CV_TypeIndexSource s, CV_TypeIndex ti) +{ + U64 lo = g->min[s]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[s]) { g->mark[s][idx] = 1; } } +} + +// walk a record's type-index sites; mark roots (do_rewrite==0) or rewrite to compacted indices (==1) +internal void +lnk_gc_visit_offsets(LNK_GCTypes *g, String8 data, CV_TiOffsets ti_offs) +{ + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&ti_offs, ti_idx); + U8 *p = data.str + n.offset; + CV_TypeIndex ti = memory_read32(p); + if (g->do_rewrite) { + U64 lo = g->min[n.source]; + if (ti >= lo) { U64 idx = ti - lo; if (idx < g->orig_n[n.source]) { memory_write32(p, g->remap[n.source][idx]); } } + } else { + lnk_gc_mark_ti(g, n.source, ti); + } + } +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_syms_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->sym_ranges[task_id]) { + LNK_SymbolInput symbols = g->cv->symbol_inputs[i]; + for (U64 cursor = 0; cursor + sizeof(CV_SymbolHeader) <= symbols.raw_symbols.size; ) { + CV_Symbol symbol = {0}; + TryReadBreak(cv_read_symbol(symbols.raw_symbols, cursor, CV_SymbolAlign, &symbol), cursor); + lnk_gc_visit_offsets(g, symbol.data, cv_symbol_ti_offsets(symbol.kind, symbol.data)); + } + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_inlines_task) +{ + LNK_GCTypes *g = raw_task; + U64 obj_idx = task_id; + Temp scratch = scratch_begin(0, 0); + String8List inlinee_lines = cv_sub_section_from_debug_s(g->cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_InlineeLines); + for EachNode(dn, String8Node, inlinee_lines.first) { + Temp temp = temp_begin(scratch.arena); + CV_TiOffsets l = cv_inlinee_ti_offsets(temp.arena, dn->string); + lnk_gc_visit_offsets(g, dn->string, l); + temp_end(temp); + } + scratch_end(scratch); +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_rewrite_leaves_task) +{ + LNK_GCTypes *g = raw_task; + Temp scratch = scratch_begin(0, 0); + for EachInRange(i, g->cur_ranges[task_id]) { + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->cur_leaf_v[i]); + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + lnk_gc_visit_offsets(g, leaf.data, l); + temp_end(temp); + } + scratch_end(scratch); +} + +typedef struct LNK_GCRingTask +{ + U8 **leaf_v; + Rng1U64 *ranges; + U64 *counts; + U64 *offsets; + LNK_GCNamePair *pairs; +} LNK_GCRingTask; + +// parallel: count UDT leaves with a unique_name per range (pass 0) / emit (hash,idx) pairs (pass 1) +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_count_task) +{ + LNK_GCRingTask *t = raw_task; + U64 n = 0; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { n += 1; } + } + } + t->counts[task_id] = n; +} + +internal +THREAD_POOL_TASK_FUNC(lnk_gc_ring_fill_task) +{ + LNK_GCRingTask *t = raw_task; + U64 cur = t->offsets[task_id]; + for EachInRange(i, t->ranges[task_id]) { + CV_Leaf leaf = cv_leaf_from_ptr(t->leaf_v[i]); + if (cv_is_udt(leaf.kind)) { + CV_UDTInfo ui = cv_get_udt_info(leaf.kind, leaf.data); + if (ui.props & CV_TypeProp_HasUniqueName) { + U64 h = 14695981039346656037ull; + for EachIndex(c, ui.unique_name.size) { h = (h ^ ui.unique_name.str[c]) * 0x100000001b3ull; } + t->pairs[cur].hash = h; t->pairs[cur].idx = (U32)i; cur += 1; + } + } + } +} + +// mark a leaf reachable and, if this is its first mark, append it to its source's frontier so a +// later round expands it. Atomic mark gates the append, so each leaf lands on the frontier once. +internal void +lnk_gc_mark_enqueue(LNK_GCTypes *g, CV_TypeIndexSource ns, U64 ci) +{ + if (ci >= g->orig_n[ns]) { return; } + if (g->mark[ns][ci]) { return; } // fast non-atomic skip: already reachable (the common edge) + // only the worker that wins the 0->1 transition appends, so the atomic runs once per leaf (not + // once per reference edge). + if (!ins_atomic_u8_eval_assign(&g->mark[ns][ci], 1)) { + U32 pos = ins_atomic_u32_inc_eval(g->fcount[ns]) - 1; + g->frontier[ns][pos] = (U32)ci; + } +} + +// one bulk-synchronous round of transitive closure: expand the frontier slice [round_begin, +// round_end) of cur_source -- visit each leaf and enqueue the leaves it references (and its +// unique_name UDT counterparts). Frontier-driven, so total work is O(reachable leaves), not +// O(rounds * total leaves). +internal +THREAD_POOL_TASK_FUNC(lnk_gc_expand_task) +{ + LNK_GCTypes *g = raw_task; + CV_TypeIndexSource s = g->cur_source; + Temp scratch = scratch_begin(0, 0); + for EachInRange(local, g->cur_ranges[task_id]) { + U32 i = g->frontier[s][g->round_begin + local]; + + Temp temp = temp_begin(scratch.arena); + CV_Leaf leaf = cv_leaf_from_ptr(g->leaf_v[s][i]); + CV_TiOffsets l = cv_leaf_ti_offsets(temp.arena, leaf.kind, leaf.data); + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&l); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff n = cv_ti_offset_at(&l, ti_idx); + CV_TypeIndex ti = memory_read32(leaf.data.str + n.offset); + U64 lo = g->min[n.source]; + if (ti >= lo) { lnk_gc_mark_enqueue(g, n.source, ti - lo); } + } + temp_end(temp); + + if (s == CV_TypeIndexSource_TPI) { + for (U32 j = g->udt_next[i]; j != i; j = g->udt_next[j]) { + lnk_gc_mark_enqueue(g, CV_TypeIndexSource_TPI, j); + } + } + } + scratch_end(scratch); +} + +internal void +lnk_gc_types(TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + + LNK_GCTypes g = {0}; + g.cv = cv; + U64 total_leaves = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.min[s] = types->min_type_indices[s]; + g.orig_n[s] = types->count[s]; + g.leaf_v[s] = types->v[s]; + g.mark[s] = push_array(scratch.arena, U8, g.orig_n[s] ? g.orig_n[s] : 1); // zeroed + total_leaves += g.orig_n[s]; + } + + // mark roots: every type index referenced by a surviving symbol / inlinee record + g.do_rewrite = 0; + g.sym_ranges = tp_divide_work(scratch.arena, cv->symbol_input_count, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + + // link UDT leaves that share a unique_name into rings, so that marking any one (e.g. a + // forward ref reached as a member-pointer target) also keeps its full definition -- needed + // for the debugger to complete types referenced only by name. TPI only (IPI has no UDTs). + U64 n_tpi = g.orig_n[CV_TypeIndexSource_TPI]; + g.udt_next = push_array_no_zero(scratch.arena, U32, n_tpi ? n_tpi : 1); + for EachIndex(i, n_tpi) { g.udt_next[i] = (U32)i; } + { + LNK_GCRingTask rt = {0}; + rt.leaf_v = g.leaf_v[CV_TypeIndexSource_TPI]; + rt.ranges = tp_divide_work(scratch.arena, n_tpi, tp->worker_count); + rt.counts = push_array(scratch.arena, U64, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_count_task, &rt); + rt.offsets = offsets_from_counts_array_u64(scratch.arena, rt.counts, tp->worker_count); + U64 np = sum_array_u64(tp->worker_count, rt.counts); + rt.pairs = push_array_no_zero(scratch.arena, LNK_GCNamePair, np ? np : 1); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_ring_fill_task, &rt); + + radsort(rt.pairs, np, lnk_gc_name_pair_is_before); + for (U64 a = 0; a < np; ) { + U64 b = a + 1; + while (b < np && rt.pairs[b].hash == rt.pairs[a].hash) { b += 1; } + for (U64 k = a; k < b; k += 1) { g.udt_next[rt.pairs[k].idx] = rt.pairs[(k + 1 < b) ? (k + 1) : a].idx; } + a = b; + } + } + + // transitive closure (parallel, bulk-synchronous): repeat rounds that visit each + // marked-but-unexpanded leaf and mark what it references, until a round marks nothing new + // seed the frontier with the root-marked leaves (one O(total leaves) scan), then expand + // frontier slices until both sources drain. Each leaf is expanded exactly once. + U32 fcount[CV_TypeIndexSource_COUNT] = {0}; + U64 start [CV_TypeIndexSource_COUNT] = {0}; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.frontier[s] = push_array_no_zero(scratch.arena, U32, g.orig_n[s] ? g.orig_n[s] : 1); + g.fcount[s] = &fcount[s]; + for EachIndex(i, g.orig_n[s]) { if (g.mark[s][i]) { g.frontier[s][fcount[s]++] = (U32)i; } } + } + for (;;) { + B32 any = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + U64 begin = start[s], end = fcount[s]; // fcount may grow during the round (cross-source enqueues) + if (begin < end) { + any = 1; + g.cur_source = (CV_TypeIndexSource)s; + g.round_begin = begin; + g.round_end = end; + g.cur_ranges = tp_divide_work(scratch.arena, end - begin, tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_expand_task, &g); + start[s] = end; + } + } + if (!any) { break; } + } + + // compact each source: assign new contiguous type indices to the kept leaves. The leaf + // pointer array is compacted IN PLACE (kept count only shrinks, so the write cursor never + // passes the read cursor) and remap lives in scratch -- so the GC adds nothing to the arena + // that survives into the (peak) PDB build. + U64 kept_total = 0; + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.remap[s] = push_array_no_zero(scratch.arena, CV_TypeIndex, g.orig_n[s] ? g.orig_n[s] : 1); + U8 **v = g.leaf_v[s]; + U64 new_n = 0; + for EachIndex(idx, g.orig_n[s]) { + if (g.mark[s][idx]) { g.remap[s][idx] = (CV_TypeIndex)(g.min[s] + new_n); v[new_n++] = v[idx]; } + else { g.remap[s][idx] = 0; /* T_NOTYPE; never referenced by a kept record */ } + } + types->count[s] = new_n; + kept_total += new_n; + } + + // rewrite all type-index references to the compacted indices + g.do_rewrite = 1; + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_syms_task, &g); + tp_for_parallel(tp, 0, cv->obj_count, lnk_gc_inlines_task, &g); + for EachIndex(s, CV_TypeIndexSource_COUNT) { + g.cur_source = (CV_TypeIndexSource)s; + g.cur_leaf_v = types->v[s]; + g.cur_ranges = tp_divide_work(scratch.arena, types->count[s], tp->worker_count); + tp_for_parallel(tp, 0, tp->worker_count, lnk_gc_rewrite_leaves_task, &g); + } + + if (lnk_get_log_status(LNK_Log_Debug)) { + lnk_log(LNK_Log_Debug, "type GC: kept %llu of %llu leaves (pruned %llu)", kept_total, total_leaves, total_leaves - kept_total); + } + + scratch_end(scratch); + ProfEnd(); +} + +typedef struct +{ + U64 weight; + U32 obj_idx; +} LNK_ObjDistWeight; + +force_inline int +lnk_obj_dist_weight_is_before(void *raw_a, void *raw_b) +{ + LNK_ObjDistWeight *a = raw_a, *b = raw_b; + if (a->weight != b->weight) { return a->weight > b->weight; } + return a->obj_idx < b->obj_idx; // deterministic total order +} + +// FAIR-SHARE: distribute cv->obj_count objs across `worker_count` lane buckets. +// Rebuilt per barrier pass so the distribution matches the cohort C that pass +// runs at (lnk_move_global_symbols_to_gsi / lnk_write_pdb_modules read +// task->obj_indices[task_id] for lanes [0,C)). Output is width- and +// assignment-independent -- per-obj results land in per-obj slots (module +// streams) or in GSI bucket chains that are content-sorted at serialization +// (gsi_symbol_is_before radsorts every chain) -- so any deterministic partition +// produces byte-identical PDB bytes; only the per-lane balance changes. +// +// `weights` (optional, [obj_count]) upgrades the round-robin to a greedy LPT +// (longest-processing-time) assignment: objs are taken in weight-descending +// order (obj_idx tie-break -> deterministic) and each goes to the least-loaded +// lane. Round-robin ignores per-obj symbol-stream size, so a lane that draws +// several giant objs holds the whole barrier pass at the final barrier while +// the other lanes idle. +internal void +lnk_build_pdb_distribute_obj_indices(Arena *arena, LNK_BuildPdb *task, U64 obj_count, U32 worker_count, U64 *weights) +{ + task->obj_indices = push_array(arena, U32Array, worker_count); + if (weights == 0) { + U64 objs_per_worker = CeilIntegerDiv(obj_count, worker_count); + for EachIndex(i, worker_count) { task->obj_indices[i].v = push_array(arena, U32, objs_per_worker ? objs_per_worker : 1); } + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[obj_idx % worker_count]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + } else { + Temp scratch = scratch_begin(&arena, 1); + + LNK_ObjDistWeight *order = push_array_no_zero(scratch.arena, LNK_ObjDistWeight, obj_count); + for EachIndex(obj_idx, obj_count) { order[obj_idx] = (LNK_ObjDistWeight){ .weight = weights[obj_idx], .obj_idx = (U32)obj_idx }; } + radsort(order, obj_count, lnk_obj_dist_weight_is_before); + + U64 *loads = push_array(scratch.arena, U64, worker_count); + U32 *assign = push_array_no_zero(scratch.arena, U32, obj_count); + for EachIndex(i, obj_count) { + U32 min_lane = 0; + for (U32 lane = 1; lane < worker_count; lane += 1) { if (loads[lane] < loads[min_lane]) { min_lane = lane; } } + assign[order[i].obj_idx] = min_lane; + loads[min_lane] += order[i].weight + 1; // +1 spreads zero-weight objs too + task->obj_indices[min_lane].count += 1; + } + + for EachIndex(lane, worker_count) { + task->obj_indices[lane].v = push_array_no_zero(arena, U32, task->obj_indices[lane].count); + task->obj_indices[lane].count = 0; + } + // fill in ascending obj order per lane (deterministic, cache-friendly iteration) + for EachIndex(obj_idx, obj_count) { + U32Array *obj_indices = &task->obj_indices[assign[obj_idx]]; + obj_indices->v[obj_indices->count++] = (U32)obj_idx; + } + + scratch_end(scratch); + } +} + internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags) { @@ -3274,6 +5086,13 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config builder_flags = ~0; } + // ini= bucket: pdb_alloc_ commits the MSF + type-server tables. This was + // ~132K faults/link (the entire prod #3 bucket) while commit_memory's RIO + // probe-and-lock prefaulted every committed page of the first 512MiB MSF + // page-data node; with lazy commit the bracket is ~0 -- kept for attribution + // so a regression here is visible on the summary line + lnk_summary_phase_begin(LNK_SummaryPhase_PdbIni); + LNK_BuildPdb task = { .image_data = image_data, .symtab = symtab, @@ -3310,29 +5129,32 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config // set min type indices for EachElement(ti_source, cv_types.min_type_indices) { task.pdb->type_servers[ti_source]->ti_lo = cv_types.min_type_indices[ti_source]; } - // per worker obj indices - { - U64 objs_per_worker = CeilIntegerDiv(cv->obj_count, tp->worker_count); - task.obj_indices = push_array(scratch.arena, U32Array, tp->worker_count); - for EachIndex(i, tp->worker_count) { task.obj_indices[i].v = push_array(scratch.arena, U32, objs_per_worker); } - for EachIndex(obj_idx, cv->obj_count) { - U32Array *obj_indices = &task.obj_indices[obj_idx % tp->worker_count]; - obj_indices->v[obj_indices->count++] = obj_idx; - } - } + // per-worker obj indices are (re)distributed per barrier pass to the cohort + // that pass actually runs at (FAIR-SHARE: tp->worker_count is pinned to the + // cohort C inside each tp_barrier_begin/end bracket, and the lnk_move_global_ + // symbols_to_gsi / lnk_write_pdb_modules tasks read task.obj_indices[task_id] + // for lanes [0,C)). Distributing to the full worker_count up front would leave + // objs in buckets [C,worker_count) unprocessed when Ctype_servers[CV_TypeIndexSource_IPI], cv_types.count[CV_TypeIndexSource_IPI], cv_types.v[CV_TypeIndexSource_IPI]); } if (builder_flags & LNK_PDB_BuilderFlag_Tpi) { pdb_type_server_push_parallel(tp, task.pdb->type_servers[CV_TypeIndexSource_TPI], cv_types.count[CV_TypeIndexSource_TPI], cv_types.v[CV_TypeIndexSource_TPI]); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbTpi); + lnk_summary_phase_begin(LNK_SummaryPhase_PdbStr); ProfBegin("Merge String Tables"); task.string_ht = cv_dedup_string_tables(tp_arena, tp, cv->obj_count, cv->debug_s_arr); cv_string_hash_table_assign_buffer_offsets(tp, task.string_ht); ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbStr); task.string_table_base_offset = task.pdb->info->strtab.size; ProfBegin("Add string tables"); @@ -3346,14 +5168,47 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config task.mod_arr[obj_idx] = dbi_push_module(task.pdb->dbi, cv->obj_arr[obj_idx]->path, lnk_obj_get_lib_path(cv->obj_arr[obj_idx])); } - ProfScope("Write Modules") tp_for_parallel(tp, 0, tp->worker_count, lnk_write_pdb_modules, &task); + ProfScope("Write Modules") + { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbGsi); + U64 phase_begin_us = now_time_us(); + // FAIR-SHARE: pin the cohort, distribute objs over exactly the cohort lanes, + // then run the barrier pass at that cohort. tp_barrier_begin sets + // tp->worker_count := C for the bracket. + U32 C = tp_barrier_begin(tp); + // weight = total debug$S byte size: the module-stream write walks every + // subsection of the obj (symbols + lines + checksums + ...) + U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); + for EachIndex(obj_idx, cv->obj_count) { + U64 total = 0; + for EachIndex(k, CV_C13SubSectionIdxKind_COUNT) { total += cv->debug_s_arr[obj_idx].data_list[k].total_size; } + weights[obj_idx] = total; + } + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); + tp_for_parallel_reserve(tp, 0, C, lnk_write_pdb_modules, &task); // BARRIER pass (path B): barrier_wait/tp_broadcast + tp_barrier_end(tp); + lnk_log(LNK_Log_Timers, "[pdb] write modules in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + lnk_summary_phase_end(LNK_SummaryPhase_PdbMod); + } if (output_ptr != 0) { for EachIndex(obj_idx, cv->obj_count) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.mod_arr[obj_idx]->sn); } } - ProfScope("Move Global Symbols") tp_for_parallel(tp, 0, tp->worker_count, lnk_move_global_symbols_to_gsi, &task); - ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); + ProfScope("Move Global Symbols") + { + U64 phase_begin_us = now_time_us(); + U32 C = tp_barrier_begin(tp); + // weight = per-obj symbols-subsection byte size: both proc-refs passes walk + // exactly these bytes per obj, and String8List.total_size makes it O(1) + U64 *weights = push_array_no_zero(scratch.arena, U64, cv->obj_count); + for EachIndex(obj_idx, cv->obj_count) { weights[obj_idx] = cv_sub_section_from_debug_s(cv->debug_s_arr[obj_idx], CV_C13SubSectionKind_Symbols).total_size; } + lnk_build_pdb_distribute_obj_indices(scratch.arena, &task, cv->obj_count, C, weights); + tp_for_parallel_reserve(tp, 0, C, lnk_move_global_symbols_to_gsi, &task); // BARRIER pass (path B): tp_sum_u64/tp_broadcast/barrier_wait + tp_barrier_end(tp); + lnk_log(LNK_Log_Timers, "[pdb] move global symbols in %.2f ms (cohort %u)", (F64)(now_time_us() - phase_begin_us) / 1000.0, C); + } + ProfScope("Build GSI and PSI") pdb_build_gsi_psi(tp, task.pdb); if (output_ptr != 0) { lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->publics_sn); lnk_pdb_output_enqueue_stream(output_ptr, task.pdb->msf, task.pdb->dbi->globals_sn); @@ -3362,6 +5217,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config } if (builder_flags & LNK_PDB_BuilderFlag_SC) { + lnk_summary_phase_begin(LNK_SummaryPhase_PdbSc); ProfBegin("Build Section Contrib Map"); { ProfBegin("Build DBI Section Headers"); @@ -3387,6 +5243,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config dbi_sec_list_concat_arr(&task.pdb->dbi->sec_contrib_list, cv->obj_count, task.sc_list); } ProfEnd(); + lnk_summary_phase_end(LNK_SummaryPhase_PdbSc); } if (builder_flags & LNK_PDB_BuilderFlag_NATVIS) { @@ -3421,6 +5278,7 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config ProfEnd(); } + lnk_summary_phase_begin(LNK_SummaryPhase_PdbMsf); pdb_build_dbi_info(tp, task.pdb, task.string_ht, 0, cv->is_stripped, &build_hooks); MSF_Error msf_err = msf_build(task.pdb->msf); @@ -3439,6 +5297,8 @@ lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config if (output_ptr != 0) { lnk_background_file_writer_end_file(output_ptr->writer, output_ptr->file, artifact.data.total_size); } + lnk_summary_phase_end(LNK_SummaryPhase_PdbMsf); + // NOTE: linker is about to exit so we can skip memory release // and let windows free memory since it does this faster diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index 283648551..670653e7d 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -104,9 +104,28 @@ typedef struct U64 symbol_input_count; LNK_SymbolInput *symbol_inputs; // [symbol_input_count] - Rng1U64 *symbol_input_ranges; // [worker_count] + Rng1U64 *symbol_input_ranges; // [symbol_input_range_count] U64 symbol_patch_task_count; // LNK_SymbolInputTask *symbol_patch_task; // [symbol_patch_task_count] + // FAIR-SHARE: fixed lane count symbol_input_ranges was built for (full pool width at build + // time). Barrier passes may run at a pinned cohort C < this; they must walk lanes + // [task_id, symbol_input_range_count) strided by the cohort, NOT index by task_id alone. + U64 symbol_input_range_count; + + // IFC (header-unit debug-record) resolution: + // redirects a consuming obj's local LF_IFC_RECORD placeholder TI to a leaf in + // an injected .ifc debug-records blob "obj". Consulted first in lnk_leaf_ref_from_ti. + // key = Compose64Bit(obj_idx, local_ti) + // value = Compose64Bit(blob_obj_idx, blob_leaf_idx) + B32 has_ifc_redirects; + HashMap ifc_redirect_hm; + Rng1U64 ifc_obj_range; // [min,max) range of injected blob objs in the parallel arrays + U32Array ifc_indices; // obj indices of injected .ifc blob objs (hashed/deduped first) + // exact per-obj key filter for ifc_redirect_hm: bit set iff Compose64Bit(obj_idx, ti) was pushed. + // lets lnk_leaf_ref_from_ti skip the (miss-dominated) hash-map search entirely; on a set bit the + // original map is searched unchanged, so results are bit-identical to always searching. + U64 **ifc_redirect_bits; // [count]; null == obj has no redirect keys + Rng1U64 *ifc_redirect_ti_rng; // [count]; [min,max) local-TI span covered by the obj's bitset } LNK_CodeViewInput; typedef struct @@ -200,6 +219,22 @@ typedef struct U64 pop_obj_idx; Rng1U64 *pop_range; + // deterministic unique-leaf estimate: distinct-hash bitmaps (per ti source) filled with + // commutative atomic ORs over the precomputed debug_h hashes -> same input, same bits, same + // estimate every run. sized pow2 so bit index is hash & (bits-1). + U32 *estimate_bitmap [CV_TypeIndexSource_COUNT]; // [estimate_bitmap_bits/32] + U64 estimate_bitmap_bits[CV_TypeIndexSource_COUNT]; // pow2 + + // set when a probe wraps without finding a slot (estimate-sized table overflowed); dedup is + // retried once with the always-sufficient total-based caps. deterministic: overflow happens + // iff the unique count exceeds cap, which is a function of the input alone. + U32 leaf_ht_overflow; + + // materialize unique leaves (unbucket + leaf TI-fixup fused, applied to a private copy so the + // fixup never dirties the copy-on-write input mapping) + U64 *leaf_buffer_offsets; // [worker_count+1] per-lane byte offsets into leaf_buffer + U8 *leaf_buffer; + LNK_MergedTypes result; } LNK_MergeTypes; @@ -271,8 +306,8 @@ internal LNK_CodeViewInput lnk_make_code_view_input(TP_Context *tp, TP_Arena *tp internal int lnk_leaf_ref_compare (LNK_LeafRef a, LNK_LeafRef b); internal int lnk_leaf_ref_is_before (void *raw_a, void *raw_b); internal B32 lnk_match_leaf_ref (LNK_CodeViewInput *input, LNK_LeafRef a, LNK_LeafRef b); -internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); -internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); +internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs, B32 discard_cycles); +internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TiOffsets ti_offs); internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); internal CV_TypeIndex lnk_assigned_ti_hash_search (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); @@ -281,4 +316,5 @@ internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP //////////////////////////////// // PDB +internal void lnk_gc_types (TP_Context *tp, Arena *arena, LNK_CodeViewInput *cv, LNK_MergedTypes *types); internal LNK_FileArtifact lnk_build_pdb(TP_Context *tp, TP_Arena *tp_arena, String8 image_data, LNK_Config *config, LNK_SymbolTable *symtab, LNK_CodeViewInput *cv, LNK_MergedTypes cv_types, LNK_PdbWriter writer, LNK_PDB_BuilderFlags builder_flags); diff --git a/src/linker/lnk_io.c b/src/linker/lnk_io.c index 8e522dfbc..85dc62f86 100644 --- a/src/linker/lnk_io.c +++ b/src/linker/lnk_io.c @@ -213,6 +213,73 @@ THREAD_POOL_TASK_FUNC(lnk_data_from_file_path_task) task->data_arr.v[task_id] = str8(buffer, read_size); } +#if OS_WINDOWS +// Input views are mapped from PAGE_WRITECOPY sections with FILE_MAP_READ access, so an +// untouched view carries no pagefile commit charge. Mapping with FILE_MAP_COPY instead +// would charge commit for the ENTIRE view at map time -- even for pages never written -- +// so N concurrent big links would hold input-set-sized commit for their whole runtime +// (measured 22.4 GiB of a 49.7 GiB peak on a large editor DLL link) and feed build-farm +// memory admission limits for no benefit. +// +// The few remaining writers that patch input bytes in place (IFC 0x1522 LF_IFC_RECORD +// NOTYPE pokes, LF_ENDPRECOMP removal in debug$P; a couple of bytes per page, a handful +// of pages per link) hit this vectored handler, which promotes JUST the faulting page to +// PAGE_WRITECOPY and retries. Commit is then charged per dirtied page instead of per +// view. Write semantics are identical to the old FILE_MAP_COPY mapping: the first write +// makes the page private, the input file is never modified. Pages of ordinary allocations +// or of read-write mappings never reach the handler (they do not fault on write), so +// /RAD_MEMORY_MAP_FILES:READ_WRITE and no-map modes are unaffected. +global volatile LONG g_lnk_cow_veh_installed; +global volatile LONG g_lnk_cow_promoted_pages; + +internal LONG NTAPI +lnk_cow_page_promote_veh(EXCEPTION_POINTERS *info) +{ + EXCEPTION_RECORD *er = info->ExceptionRecord; + if (er->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && er->NumberParameters >= 2 && er->ExceptionInformation[0] == 1) { + void *addr = (void *)er->ExceptionInformation[1]; + MEMORY_BASIC_INFORMATION mbi = {0}; + if (VirtualQuery(addr, &mbi, sizeof(mbi)) >= sizeof(mbi) && mbi.Type == MEM_MAPPED) { + if (mbi.Protect == PAGE_READONLY) { + void *page = (void *)((UINT_PTR)addr & ~(UINT_PTR)(KB(4) - 1)); + DWORD old_protect = 0; + if (VirtualProtect(page, KB(4), PAGE_WRITECOPY, &old_protect)) { + InterlockedIncrement(&g_lnk_cow_promoted_pages); + return EXCEPTION_CONTINUE_EXECUTION; + } + } else if (mbi.Protect == PAGE_WRITECOPY || mbi.Protect == PAGE_READWRITE) { + // another thread promoted this page between our fault and the query; retry the write + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + +// Bulk page promotion for known hot in-place writers (obj section-header patching writes +// ~20 pages per obj; taking a VEH exception per page costs ~100us each, ~25s kernel across +// a big link). One VirtualProtect over the whole range replaces per-page exceptions. +// No-op for heap-backed inputs (MEM_PRIVATE) and for read-write shared views (their +// PAGE_READWRITE section refuses PAGE_WRITECOPY, and their pages never write-fault), +// so this is safe to call regardless of /RAD_MEMORY_MAP_FILES mode. +internal void +lnk_cow_promote_range(void *ptr, U64 size) +{ +#if OS_WINDOWS + if (size == 0) { return; } + U8 *first = (U8 *)AlignDownPow2((U64)ptr, KB(4)); + U8 *opl = (U8 *)AlignPow2((U64)ptr + size, KB(4)); + MEMORY_BASIC_INFORMATION mbi = {0}; + if (VirtualQuery(first, &mbi, sizeof(mbi)) >= sizeof(mbi) && + mbi.Type == MEM_MAPPED && + (mbi.Protect == PAGE_READONLY || mbi.Protect == PAGE_WRITECOPY)) { + DWORD old_protect = 0; + VirtualProtect(first, (U64)(opl - first), PAGE_WRITECOPY, &old_protect); + } +#endif +} + internal THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) { @@ -238,11 +305,16 @@ THREAD_POOL_TASK_FUNC(lnk_memory_map_file_task) } else { HANDLE file_handle = CreateFileW(path16.str, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file_handle != INVALID_HANDLE_VALUE) { + if (InterlockedCompareExchange(&g_lnk_cow_veh_installed, 1, 0) == 0) { + AddVectoredExceptionHandler(1, lnk_cow_page_promote_veh); + } HANDLE mapping_handle = CreateFileMappingA(file_handle, 0, PAGE_WRITECOPY, 0, 0, 0); if (mapping_handle != INVALID_HANDLE_VALUE) { LARGE_INTEGER file_size = {0}; GetFileSizeEx(file_handle, &file_size); - void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_COPY, 0, 0, file_size.QuadPart); + // FILE_MAP_READ view of a WRITECOPY section: zero commit charge at map time; + // pages become writable one at a time through lnk_cow_page_promote_veh + void *file_data = MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_size.QuadPart); if (file_data) { task->data_arr.v[task_id] = str8(file_data, file_size.QuadPart); } diff --git a/src/linker/lnk_io.h b/src/linker/lnk_io.h index dc9932b30..aff021ea7 100644 --- a/src/linker/lnk_io.h +++ b/src/linker/lnk_io.h @@ -50,6 +50,11 @@ internal File lnk_file_open_with_rename_permissions(String8 path); internal B32 lnk_file_set_delete_on_close(File handle, B32 delete_file); internal B32 lnk_file_rename(File handle, String8 new_name); +// make a range of a read-only mapped input view writable (copy-on-write) in one call; +// no-op for heap-backed or read-write-shared inputs. Writers not routed through this +// are still safe: lnk_cow_page_promote_veh promotes faulting pages one at a time. +internal void lnk_cow_promote_range(void *ptr, U64 size); + internal String8 lnk_read_data_from_file_path(Arena *arena, LNK_IO_Flags io_flags, String8 path); internal String8Array lnk_read_data_from_file_path_parallel(TP_Context *tp, Arena *arena, LNK_IO_Flags io_flags, String8Array path_arr); diff --git a/src/linker/lnk_lib.c b/src/linker/lnk_lib.c index 7f546e88f..9ffbf3661 100644 --- a/src/linker/lnk_lib.c +++ b/src/linker/lnk_lib.c @@ -20,6 +20,25 @@ lnk_first_member_sort_key_is_before(void *raw_a, void *raw_b) return str8_is_before_case_sensitive(&a->symbol_name, &b->symbol_name); } +internal force_inline U64 +lnk_symbol_name_disc(String8 name) +{ + // pack the first 8 bytes big-endian so an integer compare orders exactly like memcmp; pad short + // names with zeros. this is faithful to str8_compar_case_sensitive INCLUDING the size tie-break + // (base_strings.c: shorter prefix precedes): at the first differing padded byte either both + // strings have a real byte there (== memcmp order), or the shorter string's zero pad compares + // below the longer string's next real byte (== shorter-prefix-precedes). a zero pad byte can tie + // only with another pad byte or an embedded NUL, and any such tie leaves the discriminators + // marching in lockstep until a real difference or full equality -- so disc inequality ALWAYS + // decides the compare, and disc equality falls through to the full str8_compar. + U64 disc = 0; + U64 n = Min(name.size, 8); + for (U64 i = 0; i < n; i += 1) { + disc |= (U64)name.str[i] << (56 - i*8); + } + return disc; +} + internal B32 lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_Lib *lib_out) { @@ -70,10 +89,15 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L Assert(first_member.symbol_count == first_member.member_offset_count); symbol_count = first_member.symbol_count; - - // convert big endian offsets - for (U32 offset_idx = 0; offset_idx < symbol_count; offset_idx += 1) { - first_member.member_offsets[offset_idx] = from_be_u32(first_member.member_offsets[offset_idx]); + + // convert big endian offsets on a private copy -- converting in place would dirty + // copy-on-write pages of the mapped archive (import libs put the whole offset table here) + { + U32 *member_offsets_le = push_array_no_zero(scratch.arena, U32, symbol_count); + for (U32 offset_idx = 0; offset_idx < symbol_count; offset_idx += 1) { + member_offsets_le[offset_idx] = from_be_u32(first_member.member_offsets[offset_idx]); + } + first_member.member_offsets = member_offsets_le; } // compress member offsets to match those from the second header @@ -132,7 +156,14 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L scratch_end(scratch); } - + + // build packed discriminators parallel to the (sorted) symbol name dir; bsearch probes read this + // contiguous array and touch archive string-table bytes only on discriminator ties + U64 *symbol_discs = push_array_no_zero(arena, U64, symbol_names.count); + for EachIndex(symbol_idx, symbol_names.count) { + symbol_discs[symbol_idx] = lnk_symbol_name_disc(symbol_names.v[symbol_idx]); + } + // init lib lib_out->path = push_str8_copy(arena, path); lib_out->data = data; @@ -142,6 +173,7 @@ lnk_lib_from_data(Arena *arena, String8 data, String8 path, U64 input_idx, LNK_L lib_out->member_offsets = member_offsets; lib_out->symbol_indices = symbol_indices; lib_out->symbol_names = symbol_names; + lib_out->symbol_discs = symbol_discs; lib_out->long_names = parse.long_names; lib_out->input_idx = input_idx; @@ -220,10 +252,61 @@ lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U return result; } +internal force_inline int +lnk_disc_str8_compar(U64 a_disc, String8 *a, U64 b_disc, String8 *b) +{ + // discriminator inequality decides the compare without touching string bytes (see + // lnk_symbol_name_disc for the order-fidelity argument); ties fall through to the full compare + if (a_disc != b_disc) { + return a_disc < b_disc ? -1 : +1; + } + return str8_compar_case_sensitive(a, b); +} + +// str8_array_bsearch with a packed-discriminator pre-filter: identical probe sequence and result +// (each probe's compare outcome is identical), but probes read the contiguous disc[] array instead +// of chasing symbol_names.v[].str into scattered archive string-table bytes +internal U64 +lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value) +{ + String8Array arr = lib->symbol_names; + U64 *disc = lib->symbol_discs; + if (arr.count > 1) { + U64 value_disc = lnk_symbol_name_disc(value); + + int lo_compar = lnk_disc_str8_compar(value_disc, &value, disc[0], &arr.v[0]); + if (lo_compar == 0) { + return 0; + } + + int hi_compar = lnk_disc_str8_compar(value_disc, &value, disc[arr.count-1], &arr.v[arr.count-1]); + if (hi_compar == 0) { + return arr.count-1; + } + + if (lo_compar > 0 && hi_compar < 0) { + for (U64 l = 0, r = arr.count-1; l <= r; ) { + U64 m = l + (r - l) / 2; + int cmp = lnk_disc_str8_compar(disc[m], &arr.v[m], value_disc, &value); + if (cmp == 0) { + return m; + } else if (cmp < 0) { + l = m + 1; + } else { + r = m - 1; + } + } + } + } else if (arr.count == 1 && str8_match(arr.v[0], value, 0)) { + return 0; + } + return max_U64; +} + internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out) { - U64 symbol_idx = str8_array_bsearch(lib->symbol_names, symbol_name); + U64 symbol_idx = lnk_lib_bsearch_symbol_name(lib, symbol_name); if (symbol_idx < lib->symbol_count) { if (member_idx_out) { *member_idx_out = lib->symbol_indices[symbol_idx]-1; diff --git a/src/linker/lnk_lib.h b/src/linker/lnk_lib.h index 6db16e293..bce3b9290 100644 --- a/src/linker/lnk_lib.h +++ b/src/linker/lnk_lib.h @@ -13,6 +13,14 @@ typedef struct LNK_Lib U32 *member_offsets; U16 *symbol_indices; String8Array symbol_names; + + // symbol-dir bsearch pre-filter: symbol_names.v[].str points into the mapped archive's string + // table, so every bsearch probe's MemCompare chases scattered archive bytes (sorted order != + // memory order -> no locality). symbol_discs[i] packs the first 8 bytes of symbol_names.v[i] + // big-endian (zero-padded), so an integer compare of discriminators decides str8_compar order + // whenever they differ; probes touch archive bytes only on discriminator ties. Parallel to + // symbol_names, built once at parse time. + U64 *symbol_discs; String8 long_names; U64 input_idx; @@ -70,5 +78,6 @@ internal LNK_Lib ** lnk_array_from_lib_list(Arena *arena, LNK_LibList list internal void lnk_lib_list_push_node(LNK_LibList *list, LNK_LibNode *node); internal LNK_LibNodeArray lnk_lib_list_push_parallel(TP_Context *tp, TP_Arena *arena, LNK_LibList *list, U64 inputs_count, struct LNK_Input **inputs); +internal U64 lnk_lib_bsearch_symbol_name(LNK_Lib *lib, String8 value); internal force_inline B32 lnk_search_lib(LNK_Lib *lib, String8 symbol_name, U32 *member_idx_out); diff --git a/src/linker/lnk_log.c b/src/linker/lnk_log.c index 5dce5875d..d8700d3c1 100644 --- a/src/linker/lnk_log.c +++ b/src/linker/lnk_log.c @@ -28,6 +28,9 @@ lnk_fprintf(FILE *f, char *fmt, ...) internal void lnk_exit(int code) { + // one-line summary must reach the build log on error exits too (best-effort; + // takes/drops g_log_mutex internally, so print BEFORE taking it here) + lnk_print_summary(code); mutex_take(g_log_mutex); fflush(stdout); fflush(stderr); diff --git a/src/linker/lnk_log.h b/src/linker/lnk_log.h index b10c836b5..01bf5fae2 100644 --- a/src/linker/lnk_log.h +++ b/src/linker/lnk_log.h @@ -13,7 +13,8 @@ X(SizeBreakdown) \ X(LinkStats) \ X(Timers) \ - X(Links) + X(Links) \ + X(Summary) typedef enum { diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index 73bd5366f..9356f9037 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -733,6 +733,9 @@ lnk_coff_string_table_from_obj(LNK_Obj *obj) internal String8 lnk_coff_symbol_table_from_obj(LNK_Obj *obj) { + if (obj->symbol_table_copy.size) { + return obj->symbol_table_copy; + } return str8_substr(obj->data, obj->header.symbol_table_range); } @@ -804,6 +807,18 @@ lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) return result; } +// Fetch section bytes for a section of an obj. Debug sections that were reloc-patched have a +// private post-fixup copy (made in lnk_obj_reloc_patcher so relocs never dirty the copy-on-write +// input mapping); prefer that copy, otherwise read straight out of the mapped input. +internal String8 +lnk_obj_get_sect_data(LNK_Obj *obj, U64 sect_idx, Rng1U64 frange) +{ + if (obj->sect_data_copies != 0 && obj->sect_data_copies[sect_idx].size != 0) { + return obj->sect_data_copies[sect_idx]; + } + return str8_substr(obj->data, frange); +} + internal THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) { @@ -819,7 +834,7 @@ THREAD_POOL_TASK_FUNC(lnk_collect_obj_chunks_task) String8 section_name = lnk_obj_section_name_from_sect_idx(obj, sect_idx); if (str8_match(section_name, task->name, 0)) { - String8 section_data = str8_substr(obj->data, section.frange); + String8 section_data = lnk_obj_get_sect_data(obj, sect_idx, section.frange); str8_list_push(arena, &task->out_lists[task_id], section_data); } } diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index 2f771741b..bd5da01bc 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -12,6 +12,17 @@ typedef struct LNK_SymbolNameCache U32 *name_sizes; } LNK_SymbolNameCache; +// /OPT:ICF fold record (one per section, indexed by section_number-1), filled at fold-apply. +// Distinguishes ICF folds from same-name COMDAT selection and /OPT:REF removal (all three end +// up LnkRemove'd with a redirected symlink, but only ICF folds join DIFFERENT-named sections, +// which is what the debug-info aliasing below needs to know). set==0 means not ICF-folded. +typedef struct LNK_ICFFold +{ + U32 leader_obj_idx; // input_idx of the leader's obj + U32 leader_sn; // leader section number + B8 set; +} LNK_ICFFold; + typedef struct LNK_Obj { String8 path; @@ -31,11 +42,31 @@ typedef struct LNK_Obj U32 *comdats; U32Node **associated_sections; LNK_ObjSymbolRef *symlinks; + LNK_ICFFold *icf_fold; // /OPT:ICF fold map (per section, sn-1 indexed); 0 if ICF off + String8 icf_file_chksms; // memoized obj-wide FILECHKSMS slice (see lnk_icf_obj_file_chksms); + B32 icf_file_chksms_init; // idempotent racy fill, flag published last + B8 *icf_lines_only; // .debug$S sections associated to an ICF-folded function: stay + // LnkRemove'd, but merge into the module remapped to the leader RVA + // (sect_idx indexed; 0 array ptr when ICF off / no folds). + // 1 = C13 Lines only (source breakpoints bind); 2 = full record + // tree (fold joins a DIFFERENT source location and has locals -- + // watch-window labels come from the right source) // link struct LNK_LibMemberRef *link_member; struct LNK_ObjNode *self; + // Private (reloc-patched) copies of .debug$* section data, indexed by sect_idx; null when the + // obj has no reloc-patched debug sections. Populated in lnk_obj_reloc_patcher so debug relocs + // never dirty the copy-on-write input mapping; debug-section readers fetch bytes through + // lnk_obj_get_sect_data which prefers the copy when present. + String8 *sect_data_copies; + + // Private copy of the COFF symbol table; set before the image symbol-patch passes so their + // section-number/value stores land here instead of copy-on-writing the input mapping. + // lnk_coff_symbol_table_from_obj prefers this when present. + String8 symbol_table_copy; + // type info U32 debug_t_sect_idx; U32 debug_p_sect_idx; @@ -166,6 +197,7 @@ internal U64 lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, internal String8 lnk_obj_section_name_from_section_number(LNK_Obj *obj, U64 section_number); internal String8 lnk_obj_section_name_from_sect_idx(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx); +internal String8 lnk_obj_get_sect_data(LNK_Obj *obj, U64 sect_idx, Rng1U64 frange); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 7226ed5ea..39c197953 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -441,6 +441,7 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, LNK_SymbolHashTrie *new_trie = lnk_symbol_hash_trie_chunk_list_push(arena, chunks, 0x1000); new_trie->name = &symbol->name; new_trie->symbol = symbol; + new_trie->hash = hash; MemoryZeroArray(new_trie->child); // try to insert new node @@ -461,7 +462,8 @@ lnk_symbol_hash_trie_insert_or_replace(Arena *arena, // load current symbol String8 *curr_name = ins_atomic_ptr_eval(&curr_trie->name); - if (curr_name && str8_match(*curr_name, symbol->name, 0)) { + // fast-reject on stored hash before touching the name string (str8_match still gates) + if (curr_name && curr_trie->hash == hash && str8_match(*curr_name, symbol->name, 0)) { for (LNK_Symbol *src = symbol;;) { // try replacing current symbol with zero, otherwise loop back and retry LNK_Symbol *leader = ins_atomic_ptr_eval_assign(&curr_trie->symbol, 0); @@ -507,7 +509,7 @@ lnk_symbol_hash_trie_search(LNK_SymbolHashTrie *trie, U64 hash, String8 name) if (curr == 0) { break; } - if (curr->name && str8_match(*curr->name, name, 0)) { + if (curr->name && curr->hash == hash && str8_match(*curr->name, name, 0)) { result = curr; break; } diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index 4fe2f24cd..1b2e5827e 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -63,6 +63,10 @@ typedef struct LNK_SymbolHashTrie { String8 *name; LNK_Symbol *symbol; + // full key hash stored at insert; descent fast-rejects on hash mismatch BEFORE + // dereferencing name -> String8 -> name bytes (saves 2-3 line misses/level). + // str8_match still gates the real match, so this is fast-reject only -> byte-identical. + U64 hash; struct LNK_SymbolHashTrie *child[4]; } LNK_SymbolHashTrie; @@ -79,6 +83,10 @@ typedef struct LNK_SymbolHashTrieChunkList U64 count; LNK_SymbolHashTrieChunk *first; LNK_SymbolHashTrieChunk *last; + // false-sharing pad: symtab->chunks / search_chunks are [worker_count] arrays indexed + // [worker_id]; at 24B/entry adjacent workers share a cache line on the parallel insert. + // Pad each entry to a full 64B line so each worker owns its line. Pure layout -> byte-identical. + U8 pad_[64 - 3*8]; } LNK_SymbolHashTrieChunkList; // --- Symbol Table ------------------------------------------------------------ diff --git a/src/linker/lnk_timer.c b/src/linker/lnk_timer.c index b94817d78..748c643bb 100644 --- a/src/linker/lnk_timer.c +++ b/src/linker/lnk_timer.c @@ -3,16 +3,83 @@ global LNK_Timer g_timers[LNK_Timer_Count]; +// summary (v2): every timer/phase boundary also stamps process-wide CPU + +// fault counters, so each bucket reports wall/user/kernel/faults +global LNK_SummaryCounters g_timer_counters_begin[LNK_Timer_Count]; +global LNK_SummaryCounters g_timer_counters_end [LNK_Timer_Count]; + +internal LNK_SummaryCounters +lnk_summary_counters_now(void) +{ + LNK_SummaryCounters c = { .wall_us = now_time_us() }; +#if OS_WINDOWS + FILETIME create_ft, exit_ft, kernel_ft, user_ft; + if (GetProcessTimes(GetCurrentProcess(), &create_ft, &exit_ft, &kernel_ft, &user_ft)) { + c.user_us = (((U64)user_ft.dwHighDateTime << 32) | user_ft.dwLowDateTime) / 10; + c.kern_us = (((U64)kernel_ft.dwHighDateTime << 32) | kernel_ft.dwLowDateTime) / 10; + } + PROCESS_MEMORY_COUNTERS pmc = { (DWORD)sizeof(pmc) }; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) { + c.faults = pmc.PageFaultCount; + } +#endif + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us > b.wall_us ? a.wall_us - b.wall_us : 0; + c.user_us = a.user_us > b.user_us ? a.user_us - b.user_us : 0; + c.kern_us = a.kern_us > b.kern_us ? a.kern_us - b.kern_us : 0; + c.faults = a.faults > b.faults ? a.faults - b.faults : 0; + return c; +} + +internal LNK_SummaryCounters +lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b) +{ + LNK_SummaryCounters c; + c.wall_us = a.wall_us + b.wall_us; + c.user_us = a.user_us + b.user_us; + c.kern_us = a.kern_us + b.kern_us; + c.faults = a.faults + b.faults; + return c; +} + internal void lnk_timer_begin(LNK_TimerType timer) { - g_timers[timer].begin = now_time_us(); + g_timer_counters_begin[timer] = lnk_summary_counters_now(); + g_timers[timer].begin = g_timer_counters_begin[timer].wall_us; } internal void lnk_timer_end(LNK_TimerType timer) { - g_timers[timer].end = now_time_us(); + g_timer_counters_end[timer] = lnk_summary_counters_now(); + g_timers[timer].end = g_timer_counters_end[timer].wall_us; +} + +global LNK_SummaryCounters g_summary_phase [LNK_SummaryPhase_Count]; +global LNK_SummaryCounters g_summary_phase_start[LNK_SummaryPhase_Count]; + +internal void +lnk_summary_phase_begin(LNK_SummaryPhase phase) +{ + g_summary_phase_start[phase] = lnk_summary_counters_now(); +} + +internal void +lnk_summary_phase_end(LNK_SummaryPhase phase) +{ + // atomic adds: the Write bracket runs on the background image-write thread + LNK_SummaryCounters now = lnk_summary_counters_now(); + ins_atomic_u64_add_eval(&g_summary_phase[phase].wall_us, now.wall_us - g_summary_phase_start[phase].wall_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].user_us, now.user_us - g_summary_phase_start[phase].user_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].kern_us, now.kern_us - g_summary_phase_start[phase].kern_us); + ins_atomic_u64_add_eval(&g_summary_phase[phase].faults, now.faults - g_summary_phase_start[phase].faults); } internal String8 diff --git a/src/linker/lnk_timer.h b/src/linker/lnk_timer.h index 9e5864549..55f07b1ca 100644 --- a/src/linker/lnk_timer.h +++ b/src/linker/lnk_timer.h @@ -22,3 +22,58 @@ typedef struct LNK_Timer internal void lnk_timer_begin(LNK_TimerType timer); internal void lnk_timer_end(LNK_TimerType timer); +// Per-phase counter snapshot for the end-of-link summary line (v2). Each +// boundary stamp is wall (QPC) + process-wide user/kernel CPU (GetProcessTimes) +// + process-wide soft+hard fault count (GetProcessMemoryInfo.PageFaultCount): +// 2 syscalls per boundary, negligible. Deltas are PROCESS-WIDE, so a phase that +// overlaps a concurrent thread's work (e.g. the image-write thread overlapping +// the debug-info phases) counts that work too -- attribution, not accounting. +typedef struct LNK_SummaryCounters +{ + U64 wall_us; + U64 user_us; // process user CPU, all threads + U64 kern_us; // process kernel CPU, all threads + U64 faults; // process page faults (soft+hard) +} LNK_SummaryCounters; + +internal LNK_SummaryCounters lnk_summary_counters_now(void); +internal LNK_SummaryCounters lnk_summary_counters_sub_sat(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field saturating a-b +internal LNK_SummaryCounters lnk_summary_counters_add(LNK_SummaryCounters a, LNK_SummaryCounters b); // per-field a+b + +// Phase accumulators for the end-of-link summary line. Unlike LNK_Timer +// (single begin/end shot), these ACCUMULATE across repeated brackets (e.g. +// lnk_load_inputs runs once per input round; the PDB sub-phases run again for +// the /PDBSTRIPPED build). Always measured. Image/Debug/PDB/RDI buckets come +// from g_timers (which stamp the same counters); these cover the phases that +// had no timer, plus the dbg/pdb sub-buckets. +typedef enum LNK_SummaryPhase +{ + LNK_SummaryPhase_Input, // lnk_load_inputs (parse/load objs+libs), all rounds + LNK_SummaryPhase_Resolve, // lnk_link_inputs minus contained Input time (lib search + member resolution) + LNK_SummaryPhase_Icf, // lnk_opt_icf + LNK_SummaryPhase_Ref, // lnk_opt_ref + LNK_SummaryPhase_Write, // image write thread (overlaps debug info) + + // dbg umbrella sub-buckets (printed as dbgg[...]) + LNK_SummaryPhase_DbgMcvi, // lnk_make_code_view_input + LNK_SummaryPhase_DbgMerge, // lnk_merge_types + + // pdb sub-buckets (printed as pdbg[...]); brackets sit on the pre-existing + // Prof/timer boundaries inside lnk_build_pdb + the write at its call site + LNK_SummaryPhase_PdbHsh, // lnk_replace_type_names_with_hashes (/RAD_PDB_HASH_TYPE_NAMES): parallel rewrite touching every merged TPI leaf -- storm re-fault amplifier + LNK_SummaryPhase_PdbIni, // lnk_build_pdb task init: pdb_alloc_ (MSF + type-server tables; was ~132K faults/link while commit_memory prefaulted every committed page -- lazy commit dropped it to ~0) + LNK_SummaryPhase_PdbGsi, // lnk_move_global_symbols_to_gsi barrier pass ("Move Global Symbols") + LNK_SummaryPhase_PdbSym, // pdb_build_gsi_psi ("Build GSI and PSI": symrec + GSI/PSI hash streams) + LNK_SummaryPhase_PdbMod, // lnk_write_pdb_modules barrier pass ("Write Modules") + LNK_SummaryPhase_PdbTpi, // pdb_type_server_push_parallel TPI+IPI + pdb_type_server_build TPI/IPI + LNK_SummaryPhase_PdbStr, // string tables: cv_dedup_string_tables + offset assign + strtab add ("Merge String Tables"/"Add string tables") + LNK_SummaryPhase_PdbSc, // "Build Section Contrib Map" (per-obj section contribs + DBI section headers) + LNK_SummaryPhase_PdbMsf, // dbi_build + pdb_info_build + msf_build + page-node gather + LNK_SummaryPhase_PdbWr, // PDB file write (lnk_write_data_list_to_file_path in lnk_io) + + LNK_SummaryPhase_Count +} LNK_SummaryPhase; + +internal void lnk_summary_phase_begin(LNK_SummaryPhase phase); +internal void lnk_summary_phase_end(LNK_SummaryPhase phase); + diff --git a/src/linker/pdb_ext/pdb_builder.c b/src/linker/pdb_ext/pdb_builder.c index 06c1ddb3f..311a0ca38 100644 --- a/src/linker/pdb_ext/pdb_builder.c +++ b/src/linker/pdb_ext/pdb_builder.c @@ -1349,14 +1349,15 @@ pdb_load_types_from_leaf_list(PDB_TypeServer **type_server_arr, CV_LeafList leaf // get offsets for type indices in data blob CV_Leaf *leaf = &node->data; - CV_TypeIndexInfoList ti_info_list = cv_get_leaf_type_index_offsets(temp.arena, leaf->kind, leaf->data); - - for (CV_TypeIndexInfo *ti_info = ti_info_list.first; ti_info != 0; ti_info = ti_info->next) { - Assert(ti_info->offset + sizeof(CV_TypeIndex) <= leaf->data.size); - CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info->offset); + CV_TiOffsets ti_offs = cv_leaf_ti_offsets(temp.arena, leaf->kind, leaf->data); + + for (U64 ti_idx = 0, ti_count = cv_ti_offsets_count(&ti_offs); ti_idx < ti_count; ti_idx += 1) { + CV_TiOff ti_info = cv_ti_offset_at(&ti_offs, ti_idx); + Assert(ti_info.offset + sizeof(CV_TypeIndex) <= leaf->data.size); + CV_TypeIndex *ti_ptr = (CV_TypeIndex *)(leaf->data.str + ti_info.offset); CV_TypeIndex external_ti = *ti_ptr; - - B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info->source]; + + B32 is_complex_type = external_ti >= ti_map->min_itype[ti_info.source]; if (is_complex_type) { // search external type index CV_TypeIndex internal_tpi_idx = pdb_type_index_map_search(ti_map, CV_TypeIndexSource_TPI, external_ti); diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 3dc2c1467..151edc5d3 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -1,6 +1,30 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) +// +// DUAL-PATH thread pool. +// +// NON-SHARED mode (no /RAD_SHARED_THREAD_POOL): UPSTREAM's barrier +// implementation, VERBATIM. Workers park on a kernel barrier between passes +// (zero-syscall steady state, no per-pass semaphore traffic). tp_run_tasks +// brackets the work loop in barrier_wait at entry+exit; tp_worker_main loops +// calling it; tp_for_parallel just inits state and joins as worker 0. No +// governor thread, no budget/wake/governor semaphores, no main_semaphore. +// +// SHARED mode (/RAD_SHARED_THREAD_POOL): OUR cross-process governor. Workers +// are PARKED on wake_semaphore and woken one-per-grant; a per-process +// governor thread borrows global budget slots and wakes parked workers; +// completion is signalled via main_semaphore; path-B barrier passes run a +// fair-share cohort (tp_barrier_begin/end + tp_for_parallel_reserve). +// +// Dispatch and the worker entry point branch on pool->is_shared (== name.size>0). +// All governor/budget state is allocated only when is_shared, so non-shared mode +// has zero extra threads and zero extra synchronization objects vs upstream. +// + +//////////////////////////////////////////////////////////////////////////////// +//~ NON-SHARED (upstream barrier) path -- VERBATIM from origin/dev. + internal void tp_run_tasks(TP_Context *pool, TP_Worker *worker) { @@ -36,60 +60,336 @@ tp_worker_main(void *raw_worker) } } +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode governor stats (summary line). Zero cost when the pool is off: +// every call site below is on a shared-mode-only path. + +global TP_SharedStats g_tp_shared_stats; + +internal void +tp_stats_level_add(S64 delta) +{ + // transitions are rare (one per grant/release, thousands per link), so a tiny + // spinlock around the integrator is cheaper than any clever lock-free scheme + for (; ins_atomic_u64_eval_cond_assign(&g_tp_shared_stats.lock, 1, 0) != 0; ) { } + U64 now_us = now_time_us(); + g_tp_shared_stats.area_us += (U64)(g_tp_shared_stats.level * (S64)(now_us - g_tp_shared_stats.last_us)); + g_tp_shared_stats.last_us = now_us; + g_tp_shared_stats.level += delta; + ins_atomic_u64_eval_assign(&g_tp_shared_stats.lock, 0); +} + +internal void +tp_stats_park_add(U64 worker_us) +{ + ins_atomic_u64_add_eval(&g_tp_shared_stats.park_us, worker_us); +} + +internal void +tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out) +{ + *grant_avg_out = 0; + *park_seconds_out = 0; + if (g_tp_shared_stats.begin_us != 0) { + tp_stats_level_add(0); // finalize the integral up to now + U64 wall_us = g_tp_shared_stats.last_us - g_tp_shared_stats.begin_us; + if (wall_us > 0) { + *grant_avg_out = (F64)g_tp_shared_stats.area_us / (F64)wall_us; + } + *park_seconds_out = (F64)g_tp_shared_stats.park_us / 1000000.0; + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED-mode cross-process attach counter (summary line procs=). Counter +// SEMAPHORE, not a named section -- see the ".nproc.v3" comment in +// thread_pool.h for why (UBA virtualizes named sections per-process). + +global Semaphore g_tp_procs_sem; // zero handle = not attached +global U32 g_tp_procs_maxseen; // process-local max of observed n + +internal void +tp_procs_attach(Arena *scratch_arena, String8 name) +{ + String8 sem_name = push_str8f(scratch_arena, "%S.nproc." TP_NPROC_V, name); + Semaphore sem = semaphore_alloc(0, TP_NPROC_MAX, sem_name); // create-or-open, count starts at 0 + if (sem.u64[0] == 0) { + return; // best-effort: no semaphore, procs= prints 0/0 + } + U32 prev = 0; + if (!semaphore_drop_prev(sem, &prev)) { // attach: hold one permit + semaphore_release(sem); + return; + } + g_tp_procs_sem = sem; + g_tp_procs_maxseen = prev + 1; +} + +internal void +tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out) +{ + *attached_out = 0; + *maxseen_out = 0; + if (g_tp_procs_sem.u64[0] != 0) { + U32 prev = 0; + if (semaphore_drop_prev(g_tp_procs_sem, &prev)) { // read: +1 ... + semaphore_take(g_tp_procs_sem, 0); // ... then undo (0-timeout, count > 0 by construction) + // prev = count BEFORE the transient release = #attached, which already + // includes THIS process's attach permit -- no +1 (unlike attach) + U32 n = prev; + g_tp_procs_maxseen = Max(g_tp_procs_maxseen, n); + *attached_out = n; + } + *maxseen_out = g_tp_procs_maxseen; + } +} + +internal void +tp_procs_detach(void) +{ + if (g_tp_procs_sem.u64[0] != 0) { + semaphore_take(g_tp_procs_sem, 0); // give the attach permit back (0-timeout: never block an exit path) + semaphore_release(g_tp_procs_sem); + MemoryZeroStruct(&g_tp_procs_sem); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ SHARED (cross-process governor) path -- OURS. + +internal void +tp_for_parallel_init_state(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + pool->task_arena = task_arena; + pool->task_func = task_func; + pool->task_data = task_data; + pool->task_count = task_count; + pool->task_done = 0; + ins_atomic_u64_eval_assign(&pool->task_left, task_count); +} + +// +// SHARED work loop. Semaphore-completion model (no barrier bracket): pure +// work-stealing on the atomic task_left decrement; the last finisher pings +// main_semaphore so the dispatching main thread can return. +// +internal void +tp_run_tasks_shared(TP_Context *pool, TP_Worker *worker) +{ + for (;;) { + S64 task_left = ins_atomic_u64_dec_eval(&pool->task_left); + + // are there any tasks left to run? + if (task_left < 0) { + break; + } + + // run task + Arena *arena = pool->task_arena ? pool->task_arena->v[worker->id] : 0; + U64 task_id = pool->task_count - (task_left+1); + pool->task_func(arena, worker->id, task_id, pool->task_data, pool); + + // cache task count so we dont touch pool memory after atomic inc + U64 task_count = pool->task_count; + + // on last task ping main thread (main_semaphore is null when worker_count==1, + // in which case main runs everything inline and never waits) + U64 task_done = ins_atomic_u64_inc_eval(&pool->task_done); + if (task_done == task_count && pool->worker_count > 1) { + semaphore_drop(pool->main_semaphore); + } + } +} + +// +// SHARED worker. Parked on wake_semaphore. Woken one-per-grant. Two wake kinds: +// - path A (barrier-free, governor-driven): the worker was woken because the +// governor acquired a global budget slot for it. When the worker drains +// (tp_run_tasks_shared returns), it RETURNS that slot: release(budget) + +// granted--, so the slot can flow to another process mid-pass. +// - path B (barrier pass): the dispatching thread reserved cohort slots up +// front and woke exactly the cohort's workers. The worker just runs the pass +// and re-parks; the dispatcher releases the slots in bulk afterwards. The +// worker must NOT touch the budget here (cohort must stay live for the pass). +// internal void tp_worker_main_shared(void *raw_worker) { TP_Worker *worker = raw_worker; TP_Context *pool = worker->pool; for (; pool->is_live; ) { - if (semaphore_take(pool->exec_semaphore, max_U64)) { - tp_run_tasks(pool, worker); + if (!semaphore_take(pool->wake_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + // capture pass kind at wake time (only one pass kind is active at once) + B32 barrier_pass = pool->barrier_pass; + + tp_run_tasks_shared(pool, worker); + + if (!barrier_pass) { + // path A: hand my budget slot back so another process can use it + tp_stats_level_add(-1); + ins_atomic_u64_dec_eval(&pool->granted); + semaphore_drop(pool->budget_semaphore); } } } -internal TP_Context * +// +// Per-process governor. Sleeps until main signals a path-A pass is active, then +// acquires global budget slots (only while THIS process has pending demand) and +// wakes one local parked worker per slot. Slots are returned by the workers +// themselves when they drain, so the governor only ever ACQUIRES. +// +internal void +tp_governor_main(void *raw_pool) +{ + TP_Context *pool = raw_pool; + for (; pool->is_live; ) { + // wait for a pass to begin (or for shutdown) + if (!semaphore_take(pool->governor_semaphore, max_U64)) { + continue; + } + if (!pool->is_live) { + break; + } + + // Grant slots while the pass is live and there is demand. Cap total live + // grants at worker_count-1 (main/worker 0 is the worker_count-th runner and + // never consumes a slot). `granted` is decremented by workers as they drain. + for (; ins_atomic_u32_eval(&pool->pass_active); ) { + S64 task_left = ins_atomic_u64_eval((U64 *)&pool->task_left); + S64 demand = task_left > 0 ? task_left : 0; + S64 cap = (S64)pool->worker_count - 1; + S64 live = ins_atomic_u64_eval((U64 *)&pool->granted); + S64 want = Min(cap, demand) - live; + + if (want > 0) { + // Bounded wait so we re-check pass_active/demand and never block forever + // on budget that may never free if the pass ends first. + U64 wait_begin_us = now_time_us(); + B32 got_slot = semaphore_take(pool->budget_semaphore, wait_begin_us + 1000); + // stats: while we waited here, `want` runnable workers sat parked on budget + tp_stats_park_add((now_time_us() - wait_begin_us) * (U64)want); + if (got_slot) { + // Publish the grant (granted++) BEFORE checking pass_active, and ABORT + // with granted-- if the pass already ended. This makes main's path-A + // drain-spin (waits granted==0) observe any in-flight grant and block + // until the governor resolves it -- so main cannot exit tp_for_parallel + // and start a path-B barrier pass (which sets barrier_pass=1) while a + // grant is pending. Hence a woken worker ALWAYS captures barrier_pass==0 + // for a path-A grant and does its paired granted--. + // + // The earlier "check pass_active, then granted++" ordering was NOT + // atomic: the governor could pass the check, get preempted while main + // ended the pass + drained granted to 0 + started a path-B pass, then + // wake a worker that captured barrier_pass==1, skipped granted--, and + // wedged granted>0 forever (observed: main spinning in tp_for_parallel, + // all workers parked). granted++ first closes that window. + ins_atomic_u64_inc_eval(&pool->granted); + if (ins_atomic_u32_eval(&pool->pass_active)) { + tp_stats_level_add(+1); + semaphore_drop(pool->wake_semaphore); + } else { + ins_atomic_u64_dec_eval(&pool->granted); // abort: pass ended + semaphore_drop(pool->budget_semaphore); // give the slot back + } + } + } else { + // No demand right now (work drained or fully covered). Briefly idle; the + // pass-end ping will release us promptly via the outer take when we loop. + sleep_ms(0); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ Alloc / release (dual). + +internal TP_Context * tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) { ProfBeginDynamic("Alloc Thread Pool [Worker Count: %u]", worker_count); AssertAlways(worker_count > 0); - B32 is_shared = (name.size > 0); + B32 is_shared = (name.size > 0); + Temp scratch = scratch_begin(&arena, 1); + + // init pool + TP_Context *pool = push_array(arena, TP_Context, 1); + pool->run_barrier = barrier_alloc(worker_count); + pool->barrier = barrier_alloc(worker_count); + pool->is_live = 1; + pool->is_shared = is_shared; + pool->worker_count = worker_count; + pool->worker_arr = push_array(arena, TP_Worker, worker_count); // alloc semaphores - Semaphore exec_semaphore = {0}; - if (worker_count > 1) { - if (is_shared) { + if (is_shared) { + // SHARED: governor + budget + wake + completion. Only allocated here, so + // non-shared mode pays for none of it. + if (worker_count > 1) { AssertAlways(worker_count <= max_worker_count); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + + pool->main_semaphore = semaphore_alloc(0, 1, str8_zero()); + + // ONE NAMED cross-process semaphore. CreateSemaphoreW on an existing name + // returns the existing object (first process inits with this count; later + // processes attach and the supplied count is ignored by the OS), so all + // processes share one BUDGET. + // BUDGET: init=max=max_worker_count (the machine core budget). + // FAIR-SHARE: there is no longer a barrier-lock. A barrier pass (path B) + // does NOT amass the full cohort; it runs at whatever budget is free right + // now (best-effort), so multiple processes can run barrier passes + // concurrently and none can deadlock waiting to amass the machine. + // ".v2" LAYOUT-VERSION suffix: see TP_SharedBlock in thread_pool.h -- old + // exes ("%S.budget", no procs section) and new exes must never share + // kernel objects for the same pool name + String8 budget_name = push_str8f(scratch.arena, "%S.budget." TP_SHARED_V, name); + pool->budget_semaphore = semaphore_alloc(max_worker_count, max_worker_count, budget_name); + pool->max_worker_count = max_worker_count; + + // local wake/governor signalling. governor_semaphore is a 0/1 "at least one + // pending pass" flag: main pings it with semaphore_drop_if_room (a redundant + // ping while one is already pending is a harmless no-op, since the pending + // signal will make the governor re-evaluate the current pass_active anyway). + pool->wake_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + pool->governor_semaphore = semaphore_alloc(0, 1, str8_zero()); } } // pick entry point for the workers void *worker_entry = is_shared ? tp_worker_main_shared : tp_worker_main; - // init pool - TP_Context *pool = push_array(arena, TP_Context, 1); - pool->exec_semaphore = exec_semaphore; - pool->run_barrier = barrier_alloc(worker_count); - pool->barrier = barrier_alloc(worker_count); - pool->is_live = 1; - pool->worker_count = worker_count; - pool->worker_arr = push_array(arena, TP_Worker, worker_count); - // init worker data for (U64 i = 0; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->id = i; worker->pool = pool; } - + // launch worker threads for (U64 i = 1; i < worker_count; i += 1) { TP_Worker *worker = &pool->worker_arr[i]; worker->handle = thread_launch(worker_entry, worker); } - + + // launch the per-process governor (shared mode only) + if (is_shared && worker_count > 1) { + pool->governor_handle = thread_launch(tp_governor_main, pool); + } + + // stats: start the grant_avg integration window (shared mode only) + if (is_shared) { + g_tp_shared_stats.begin_us = g_tp_shared_stats.last_us = now_time_us(); + tp_procs_attach(scratch.arena, name); // summary line procs= (attach counter + peak watermark) + } + + scratch_end(scratch); ProfEnd(); return pool; } @@ -99,24 +399,45 @@ tp_release(TP_Context *pool) { pool->is_live = 0; - B32 is_shared = pool->exec_semaphore.u64[0] != 0; - if (is_shared) { - for EachIndex(i, pool->worker_count) { - semaphore_drop(pool->exec_semaphore); + if (pool->is_shared) { + if (pool->worker_count > 1) { + // wake governor so it observes !is_live and exits (a pending ping is fine) + semaphore_drop_if_room(pool->governor_semaphore); + // wake every parked worker so each observes !is_live and exits. Wakes here + // are NOT path-A grants (no budget was taken), so mark barrier_pass to keep + // workers from touching the budget on their way out. + pool->barrier_pass = 1; + for (U64 i = 1; i < pool->worker_count; i += 1) { + semaphore_drop(pool->wake_semaphore); + } + } + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); + } + if (pool->worker_count > 1) { + thread_detach(pool->governor_handle); + semaphore_release(pool->budget_semaphore); + semaphore_release(pool->wake_semaphore); + semaphore_release(pool->governor_semaphore); + semaphore_release(pool->main_semaphore); + } + } else { + // NON-SHARED: upstream verbatim. Workers are parked on the barrier; flipping + // is_live and waking the barrier lets each observe !is_live and exit. + for (U64 i = 1; i < pool->worker_count; i += 1) { + thread_detach(pool->worker_arr[i].handle); } } - for (U64 i = 1; i < pool->worker_count; i += 1) { - thread_detach(pool->worker_arr[i].handle); - } - if (is_shared) { - semaphore_release(pool->exec_semaphore); - } + barrier_release(pool->run_barrier); barrier_release(pool->barrier); MemoryZeroStruct(pool); } +//////////////////////////////////////////////////////////////////////////////// +//~ Arenas / temps -- shared by both modes (unchanged). + internal TP_Arena * tp_arena_alloc(TP_Context *pool) { @@ -124,7 +445,12 @@ tp_arena_alloc(TP_Context *pool) Temp scratch = scratch_begin(0,0); Arena **arr = push_array(scratch.arena, Arena *, pool->worker_count); for (U64 i = 0; i < pool->worker_count; ++i) { - arr[i] = arena_alloc("THREAD_POOL"); + // 2MB commit quantum: these per-worker arenas take the bulk of the link's + // ~50GB of MEM_COMMIT growth; the default 64KB quantum turns that into + // ~800K NtAllocateVirtualMemory calls from 64 threads serialized on the + // process address-space lock. Slack is bounded by workers x live arenas x + // quantum (single-digit MBs per worker), far below the syscall cost. + arr[i] = arena_alloc(.commit_size = MB(2), .name = "THREAD_POOL"); } Arena **dst = push_array(arr[0], Arena *, pool->worker_count); MemoryCopy(dst, arr, sizeof(Arena*) * pool->worker_count); @@ -179,11 +505,23 @@ tp_temp_end(TP_Temp temp) ProfEnd(); } +//////////////////////////////////////////////////////////////////////////////// +//~ Dispatch (dual). + internal void tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) { - if (task_count) { - // init run + if (task_count == 0) { + return; + } + + if (!pool->is_shared) { + // + // NON-SHARED: UPSTREAM verbatim. Init state, then join the barrier as worker + // 0; the already-parked workers (looping in tp_run_tasks) rendezvous at the + // entry barrier, steal tasks, and rendezvous again at the exit barrier. No + // semaphores, no governor. + // pool->task_arena = task_arena; pool->task_func = task_func; pool->task_data = task_data; @@ -191,26 +529,258 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF pool->task_done = 0; pool->task_left = task_count; - // if we are in shared mode -> ping - if (*pool->exec_semaphore.u64) { - U64 drop_count64 = pool->worker_count - 1; - U32 drop_count = safe_cast_u32(drop_count64); - semaphore_drop_count(pool->exec_semaphore, drop_count); + // run tasks on main worker + tp_run_tasks(pool, &pool->worker_arr[0]); + return; + } + + // + // SHARED: OUR governor dispatch. + // + tp_for_parallel_init_state(pool, task_arena, task_count, task_func, task_data); + + if (pool->worker_count == 1) { + // no workers: main runs everything inline + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + return; + } + + // PATH A: barrier-free dispatch (the common case). Pure work-stealing via the + // atomic task_left decrement in tp_run_tasks_shared. Main (worker 0) ALWAYS + // runs and never consumes a global budget slot -> per-process forward-progress + // guarantee. The governor opportunistically borrows budget slots and wakes + // local parked workers; each woken worker returns its slot when it drains. + + // announce a path-A pass and let the governor recruit workers as budget frees + pool->barrier_pass = 0; + ins_atomic_u32_eval_assign(&pool->pass_active, 1); + semaphore_drop_if_room(pool->governor_semaphore); + + // main always runs (no slot consumed) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // all tasks done (last finisher pinged main_semaphore) + semaphore_take(pool->main_semaphore, max_U64); + + // End the pass so the governor stops issuing new grants. + ins_atomic_u32_eval_assign(&pool->pass_active, 0); + + // CRITICAL: before returning we must guarantee that no woken worker is still + // inside tp_run_tasks_shared. Otherwise the next pass's init_state (which resets + // task_left/task_done) would race a straggler still looping on this pass and + // corrupt the counters / lose the completion ping -> deadlock. + // + // Every governor grant is paired with exactly one wake permit and one worker + // that, on draining, does `granted--; drop(budget)`. Even grants issued in the + // tiny window before pass_active was cleared have a pending wake permit that a + // worker will consume, drain immediately (task_left<0), and account for. So + // `granted` monotonically drains to 0 once the governor has stopped; spin + // until it does. This is brief (workers see task_left<0 and exit at once). + for (; ins_atomic_u64_eval((U64 *)&pool->granted) != 0; ) { + sleep_ms(0); + } +} + +//////////////////////////////////////////////////////////////////////////////// +//~ FAIR-SHARE barrier-pass cohort bracket (SHARED path B; no-op in non-shared). +// +// A barrier pass (path B) runs at the cohort this process currently holds, NOT +// the full machine. tp_barrier_begin grabs whatever budget slots are FREE RIGHT +// NOW (best-effort, never blocking to amass), up to worker_count-1, and pins the +// pool to cohort C = 1 (main) + grabbed slots for the pass duration: +// - pool->worker_count := C (so every tp->worker_count read -- divide_work, +// lane_count, per-worker array sizing in the +// caller's setup -- sees the cohort) +// - pool->barrier := a fresh C-sized barrier (so barrier_wait/broadcast/sum +// rendezvous exactly the C participants) +// - the grabbed slots are HELD until tp_barrier_end (cohort stays live; the +// governor only touches budget during a path-A pass, which cannot overlap a +// barrier pass within this process). +// +// Deadlock-freedom: tp_barrier_begin NEVER blocks on budget. If the machine is +// busy and zero slots are free, C == 1 and the pass runs serially on main. A +// process therefore ALWAYS makes progress (>= main) and never waits to amass -> +// no starvation, no deadlock, no barrier-lock. Output is width-independent +// (proven: w1 == w64), so a cohort-C pass is byte-identical to a full-width pass. +// +// In NON-SHARED mode tp_barrier_begin/end are no-ops (return pool->worker_count / +// early-out) and tp_for_parallel_reserve degrades to the plain upstream +// full-width barrier pass via tp_for_parallel. +// +internal U32 +tp_barrier_begin(TP_Context *pool) +{ + if (!pool->is_shared || pool->worker_count == 1) { + return pool->worker_count; // no-op: non-shared / single-worker + } + if (pool->barrier_depth > 0) { + pool->barrier_depth += 1; // nested: cohort already pinned + return pool->worker_count; + } + + // best-effort grab: take as many free budget slots as we can WITHOUT blocking + // (endt_us==0 -> WaitForSingleObject(.,0) non-blocking poll). Stop at the first + // empty take or when we hold worker_count-1. + U32 want = pool->worker_count - 1; + U32 extra = 0; + for (; extra < want; ) { + if (semaphore_take(pool->budget_semaphore, 0)) { + extra += 1; + } else { + break; // no more free slots right now } + } - // run tasks on main worker - tp_run_tasks(pool, pool->worker_arr); - Assert(pool->task_done == task_count); + // FAIR-SHARE FLOOR: the cohort is pinned for the whole bracket, so a bracket + // opened at a bad instant (siblings momentarily holding the machine) would run + // a long phase at width 1-2 even after the machine empties. If the free-slot + // sweep landed below this process's fair share (machine budget / attached + // processes), keep taking with bounded waits until we reach it or the deadline + // expires. Slots flow back continuously as sibling path-A workers drain, so + // this normally fills within a few ms; if every sibling is pinned in its own + // long bracket the deadline bounds the wait and we proceed with what we hold -- + // never a deadlock, cohort >= 1 always. + if (extra < want) { + U32 procs = 0, procs_maxseen = 0; + tp_procs_snapshot(&procs, &procs_maxseen); + if (procs > 1) { + U32 fair = pool->max_worker_count / procs; + fair = Clamp(1, fair, want + 1); + if (1 + extra < fair) { + U64 deadline_us = now_time_us() + TP_BARRIER_FLOOR_WAIT_US; + for (; 1 + extra < fair; ) { + U64 now_us = now_time_us(); + if (now_us >= deadline_us) { break; } + U64 slice_us = Min(deadline_us - now_us, 5000); + if (semaphore_take(pool->budget_semaphore, now_us + slice_us)) { + extra += 1; + } + } + } + } + } + + U32 cohort = 1 + extra; // main + grabbed workers + + pool->barrier_saved_workers = pool->worker_count; + pool->barrier_cohort_extra = extra; + pool->barrier_saved = pool->barrier; + pool->barrier = barrier_alloc(cohort); + pool->worker_count = cohort; + pool->barrier_pass = 1; + pool->barrier_depth = 1; + + // stats: cohort slots are held for the whole bracket; the shortfall is the + // budget we wanted but could not grab (parked lanes while this pass runs) + pool->barrier_begin_us = now_time_us(); + pool->barrier_shortfall = want - extra; + if (extra > 0) { tp_stats_level_add((S64)extra); } + + return cohort; +} + +internal void +tp_barrier_end(TP_Context *pool) +{ + if (!pool->is_shared || pool->barrier_saved_workers == 0) { + return; // no-op: non-shared / single-worker / not in a bracket + } + pool->barrier_depth -= 1; + if (pool->barrier_depth > 0) { + return; // nested: outer bracket still owns the cohort + } + + U32 extra = pool->barrier_cohort_extra; + + // stats: release the held slots from the integral; account parked lanes + if (extra > 0) { tp_stats_level_add(-(S64)extra); } + if (pool->barrier_shortfall > 0) { + tp_stats_park_add((now_time_us() - pool->barrier_begin_us) * (U64)pool->barrier_shortfall); + } + pool->barrier_begin_us = 0; + pool->barrier_shortfall = 0; + + // restore the full-width pool + the original barrier + barrier_release(pool->barrier); + pool->barrier = pool->barrier_saved; + pool->worker_count = pool->barrier_saved_workers; + pool->barrier_pass = 0; + + pool->barrier_saved_workers = 0; + pool->barrier_cohort_extra = 0; + MemoryZeroStruct(&pool->barrier_saved); + + // hand the grabbed budget slots back to the machine + semaphore_drop_n(pool->budget_semaphore, extra); +} + +// +// PATH B: barrier-pass dispatch (fair-share). Runs the task once per lane on the +// CURRENT cohort (main + woken workers). If the caller has not already opened a +// tp_barrier_begin/end bracket, this opens one (cohort = whatever is free now), +// runs, and closes it. The passed task_count is ignored for sizing -- the pass +// always runs exactly pool->worker_count tasks (== cohort). +// +// In NON-SHARED mode this degrades to the plain upstream full-width barrier pass +// (tp_for_parallel), so non-shared barrier passes behave exactly as upstream. +// +internal void +tp_for_parallel_reserve(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskFunc *task_func, void *task_data) +{ + if (task_count == 0) { + return; + } + + if (!pool->is_shared || pool->worker_count == 1) { + // non-shared (or single worker): identical to the plain dispatch (upstream + // full-width barrier pass in non-shared mode) + tp_for_parallel(pool, task_arena, task_count, task_func, task_data); + return; + } + + // open a cohort bracket unless the caller already pinned one + B32 opened = 0; + if (pool->barrier_depth == 0) { + tp_barrier_begin(pool); + opened = 1; + } + + U32 cohort = pool->worker_count; // pinned for the whole pass + + if (cohort == 1) { + // machine fully busy: run serially on main (byte-identical -- width independent) + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + } else { + tp_for_parallel_init_state(pool, task_arena, cohort, task_func, task_data); + + // wake exactly the cohort's workers (ids 1..cohort-1). These are barrier-pass + // wakes: workers must NOT return budget on drain -- the slots are held by the + // bracket and released in tp_barrier_end so the cohort stays live for the pass. + semaphore_drop_n(pool->wake_semaphore, cohort - 1); + + // main is the cohort-th participant (lane 0) + tp_run_tasks_shared(pool, &pool->worker_arr[0]); + + // wait for the cohort to finish + semaphore_take(pool->main_semaphore, max_U64); + } + + if (opened) { + tp_barrier_end(pool); } } +//////////////////////////////////////////////////////////////////////////////// +//~ Helpers -- shared by both modes (unchanged). + internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count) { U64 per_count = CeilIntegerDiv(item_count, worker_count); Rng1U64 *range_arr = push_array_no_zero(arena, Rng1U64, worker_count + 1); for (U64 i = 0; i < worker_count; i += 1) { - range_arr[i] = rng_1u64(Min(item_count, i * per_count), + range_arr[i] = rng_1u64(Min(item_count, i * per_count), Min(item_count, i * per_count + per_count)); } @@ -250,4 +820,3 @@ tp_sum_u64(TP_Context *tp, U64 task_id, U64 v) barrier_wait(tp->barrier); return result; } - diff --git a/src/linker/thread_pool/thread_pool.h b/src/linker/thread_pool/thread_pool.h index f89f17852..d7eb23fe1 100644 --- a/src/linker/thread_pool/thread_pool.h +++ b/src/linker/thread_pool/thread_pool.h @@ -38,6 +38,28 @@ typedef struct TP_Context U64 broadcast_size; U64 sum; + // shared (cross-process) governor mode; all zero in non-shared mode + B32 is_shared; + Semaphore budget_semaphore; // NAMED: global core budget; init=max=max_worker_count + Semaphore wake_semaphore; // local: governor/dispatcher wakes one parked worker per drop + Semaphore governor_semaphore; // local: main pings governor that a path-A pass is active + Thread governor_handle; + volatile U32 pass_active; // 1 while a path-A (barrier-free) pass is in flight + volatile U32 barrier_pass; // 1 while the current wake cohort is a path-B barrier pass + volatile S64 granted; // budget slots currently held by woken path-A workers + U32 max_worker_count; // machine core budget (budget_semaphore init/max) + + // FAIR-SHARE barrier-pass cohort state (path B). A barrier pass runs at the + // cohort the governor currently allows this process to hold: main + however + // many budget slots are free RIGHT NOW (best-effort, never amassed). Pinned + // for the pass duration. See tp_barrier_begin/tp_barrier_end. + U32 barrier_depth; // >0 while inside a tp_barrier_begin/end bracket (re-entrant guard) + U32 barrier_saved_workers; // worker_count to restore at tp_barrier_end + U32 barrier_cohort_extra; // budget slots held for this barrier pass (cohort = 1 + this) + Barrier barrier_saved; // pool->barrier to restore at tp_barrier_end + U64 barrier_begin_us; // stats: bracket open time (for park accounting) + U32 barrier_shortfall; // stats: budget slots we wanted but could not grab for this pass + U32 worker_count; TP_Worker *worker_arr; @@ -57,5 +79,78 @@ internal TP_Temp tp_temp_begin(TP_Arena *arena); internal void tp_temp_end(TP_Temp temp); #define tp_for_parallel_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel(pool, arena, task_count, task_func, task_data); ProfEnd(); internal void tp_for_parallel(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +// FAIR-SHARE barrier-pass cohort bracket. Between tp_barrier_begin and +// tp_barrier_end, pool->worker_count is PINNED to the cohort this process +// currently holds (1 + budget slots free right now, capped at the full count), +// and pool->barrier is sized to that cohort. Returns the cohort count C. A caller +// that pre-distributes work by tp->worker_count (sizes per-worker arrays, builds +// divide_work ranges, sets a task-data .worker_count) MUST do that setup inside +// the bracket so it sees C, then call tp_for_parallel_reserve with task_count==C. +// Re-entrant: nested begins just return the pinned C. In non-shared mode (or +// worker_count==1) it is a no-op that returns worker_count. +internal U32 tp_barrier_begin(TP_Context *pool); +internal void tp_barrier_end(TP_Context *pool); +// Barrier-pass dispatch: task_func uses barrier_wait/tp_broadcast/tp_sum_u64. The +// cohort is whatever this process currently holds (fair-share): if not already +// inside a tp_barrier_begin/end bracket this opens one itself, runs the pass at +// the pinned cohort, and closes it. Output is width-independent so any cohort +// (down to 1 = main only) produces byte-identical results. The passed task_count +// is IGNORED for sizing; the pass always runs exactly pool->worker_count (==cohort) +// tasks, one per lane. In non-shared mode it is identical to tp_for_parallel. +internal void tp_for_parallel_reserve(TP_Context *pool, TP_Arena *arena, U64 task_count, TP_TaskFunc *task_func, void *task_data); +#define tp_for_parallel_reserve_prof(pool, arena, task_count, task_func, task_data, zone_name) ProfBegin(zone_name); tp_for_parallel_reserve(pool, arena, task_count, task_func, task_data); ProfEnd(); internal Rng1U64 * tp_divide_work(Arena *arena, U64 item_count, U32 worker_count); #define tp_broadcast(p) tp_broadcast_(tp, task_id, p, sizeof(*p)) + +// SHARED-mode governor stats (for the end-of-link summary line). All counters +// are only ever touched from shared-mode-only code paths, so the non-shared +// pool pays zero cost. `level` is the count of global budget slots this +// process currently HOLDS (path-A grants + path-B cohort extras); it is +// integrated over time (area_us = sum level x dt, QPC-stamped on every +// grant/release transition) so grant_avg = area/wall. `park_us` accumulates +// worker-microseconds spent waiting on budget while this process had pending +// demand (path A: governor budget waits x wanted-worker count; path B: pass +// duration x cohort shortfall). +typedef struct TP_SharedStats +{ + volatile U64 lock; // spinlock for the {last_us, level, area_us} integrator + U64 begin_us; // pool alloc time (grant_avg denominator start) + U64 last_us; // last transition stamp + S64 level; // budget slots currently held + U64 area_us; // integral of level over time + volatile U64 park_us; // worker-us parked on budget while work was available +} TP_SharedStats; + +internal void tp_stats_level_add(S64 delta); +internal void tp_stats_park_add(U64 worker_us); +internal void tp_stats_snapshot(F64 *grant_avg_out, F64 *park_seconds_out); + +// SHARED-mode cross-process attach counter (summary line: procs=/). +// Lives in a named counter SEMAPHORE ".nproc.v3"; the budget +// semaphore is ".budget.v2". Names carry a version suffix so an old +// radlink pointed at the SAME /RAD_SHARED_THREAD_POOL name never shares kernel +// objects with a new one: mixed old/new farms run independent pools instead of +// corrupting one. +// +// WHY A SEMAPHORE: UBA detours CreateFileMapping and VIRTUALIZES named +// sections per-process, so the previous ".procs.v2" shared-memory +// block reported procs=1/1 under UBA. Named SEMAPHORES pass through the detour +// (the budget semaphore is demonstrably shared in prod: fair grants across +// processes), so the counter is the semaphore's own count: +// attach = release(+1, &prev) -> n = prev+1 (each attached process holds one permit) +// detach = 0-timeout wait (-1) +// read = release(+1, &prev) -> n = prev (prev already counts our own +// permit), then 0-timeout wait to undo +// The transient read +1 can inflate a concurrent reader's n by 1 -- advisory +// counter, acceptable. Peak cannot be tracked exactly cross-process without +// shm, so `maxseen` is this process's local max of n observed at attach and at +// the summary read. Best-effort: a process that dies without detaching leaves +// the count high until the semaphore object itself dies with its last handle. +#define TP_SHARED_V "v2" +#define TP_NPROC_V "v3" +#define TP_NPROC_MAX (1u << 20) // far above any plausible concurrent-link count +#define TP_BARRIER_FLOOR_WAIT_US 200000 // tp_barrier_begin: max wait to reach the fair-share cohort floor + +internal void tp_procs_snapshot(U32 *attached_out, U32 *maxseen_out); // 0/0 when no shared pool +internal void tp_procs_detach(void); // give the permit back + close (idempotent) + diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index 88999e5ef..fb94e965e 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -711,6 +711,54 @@ semaphore_drop_count(Semaphore semaphore, U64 count) } } +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + // POSIX sem_t is unbounded, so a post can never be rejected for being "full". + semaphore_drop(semaphore); +} + +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + // best-effort: sem_getvalue+sem_post is not atomic (unlike win32 + // ReleaseSemaphore's lpPreviousCount); callers use this for advisory + // counters only + *prev_count_out = 0; + if(semaphore.u64[0] == 0) { return 0; } + int value = 0; + if(sem_getvalue((sem_t*)*semaphore.u64, &value) == 0 && value > 0) { *prev_count_out = (U32)value; } + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)*semaphore.u64)); + return err == 0; +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + for(U32 i = 0; i < count; i += 1) + { + if(!semaphore_take(semaphore, endt_us)) + { + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; +} + //- rjf: barriers internal Barrier diff --git a/src/pe/pe_make_import_table.c b/src/pe/pe_make_import_table.c index 8bc392035..fac25311a 100644 --- a/src/pe/pe_make_import_table.c +++ b/src/pe/pe_make_import_table.c @@ -292,10 +292,13 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, ordinal_data); str8_list_push(obj_writer->arena, &iat_sect->data, ordinal_data); } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { // put together name look up entry + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); COFF_ObjSymbol *int_symbol = coff_obj_writer_push_symbol_static(obj_writer, int_sect->name, int_sect->data.total_size, int_sect); - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); str8_list_push(obj_writer->arena, &int_sect->data, int_data); // in the file IAT mirrors ILT, dynamic linker later overwrites it with imported function addresses @@ -311,8 +314,6 @@ pe_make_import_dll_obj_static(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mach str8_list_push(obj_writer->arena, &ilt_sect->data, str8_array(import_entry, import_size)); str8_list_push(obj_writer->arena, &iat_sect->data, str8_array(import_entry, import_size)); } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; default: { InvalidPath; } break; } @@ -454,9 +455,12 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_offset, load_thunk_symbol); } } break; + case COFF_ImportBy_NameNoPrefix: + case COFF_ImportBy_Undecorate: case COFF_ImportBy_Name: { // put together name look up entry - String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, import_header.func_name); + String8 lookup_name = coff_import_lookup_name_from_import_by(import_header.func_name, import_header.import_by); + String8 int_data = coff_make_import_lookup(obj_writer->arena, import_header.hint_or_ordinal, lookup_name); U64 int_data_offset = int_sect->data.total_size; str8_list_push(obj_writer->arena, &int_sect->data, int_data); @@ -496,8 +500,6 @@ pe_make_import_dll_obj_delayed(Arena *arena, COFF_TimeStamp time_stamp, COFF_Mac coff_obj_writer_section_push_reloc_addr(obj_writer, uiat_sect, uiat_data_offset, load_thunk_symbol); } } break; - case COFF_ImportBy_Undecorate: { NotImplemented; } break; - case COFF_ImportBy_NameNoPrefix: { NotImplemented; } break; } } diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index 296373d03..594abb556 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -272,15 +272,17 @@ internal B32 commit_memory(void *ptr, U64 size) { B32 result = (VirtualAlloc(ptr, size, MEM_COMMIT, PAGE_READWRITE) != 0); - -#if !NO_WIN32_RIO - if(w32_rio_functions.RIORegisterBuffer) - { - // wine does not implement these functions - w32_rio_functions.RIODeregisterBuffer(w32_rio_functions.RIORegisterBuffer(ptr, size)); - } -#endif - + // NOTE(perf): this used to RIORegisterBuffer+RIODeregisterBuffer the committed + // range as a batched prefault trick. Registration probe-and-locks EVERY page in + // the range, which turns every commit into an eager demand-zero fault of the + // full range: pages that are never subsequently touched (e.g. the tail of a + // 512MiB MSF page-data node, oversized table slots) still get faulted, zeroed, + // and charged to the working set. On a big editor link that was ~1.2M + // process page faults and multiple GiB of resident-but-never-used memory. + // Plain MEM_COMMIT is cheap (no page is touched); pages fault in lazily on + // first touch, so the fault count tracks what is actually used and the + // faults land spread across parallel workers instead of serially at the + // commit site. #if PROFILE_TELEMETRY tmAlloc(0, ptr, size / 1024, "Win32 Commit"); #endif @@ -698,6 +700,64 @@ semaphore_drop_count(Semaphore semaphore, U64 drop_count) ReleaseSemaphore((HANDLE)*semaphore.u64, drop_count, 0); } +internal B32 +semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out) +{ + LONG prev = 0; + BOOL ok = ReleaseSemaphore((HANDLE)*semaphore.u64, 1, &prev); + *prev_count_out = ok ? (U32)prev : 0; + return !!ok; +} + +// Best-effort post: succeed if there is room, silently no-op if the count is +// already at max (ERROR_TOO_MANY_POSTS). Use ONLY for "at least one pending +// signal" wakeups (e.g. the governor ping) where redundant posts are harmless. +// Any OTHER failure is still a hard error. +internal void +semaphore_drop_if_room(Semaphore semaphore) +{ + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, 1, 0); + if (!ok) { + DWORD err = GetLastError(); + AssertAlways(err == ERROR_TOO_MANY_POSTS); + } +} + +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + BOOL ok = ReleaseSemaphore(handle, count, 0); + if (!ok) { + // The non-shared thread pool intentionally batches a wake of up to + // worker_count permits onto a semaphore that may still hold un-retaken + // permits from a prior pass; the surplus clamps at the max and the OS + // returns ERROR_TOO_MANY_POSTS. That is a benign over-wake (workers are + // already runnable), so tolerate it -- but ONLY it. Any other failure + // (e.g. a bad handle) is a real bug and must not be swallowed. + DWORD err = GetLastError(); + AssertAlways(err == ERROR_TOO_MANY_POSTS); + } + } +} + +internal B32 +semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us) +{ + // Blocking acquire of `count` permits, one at a time. Off the hot path only: + // used by the shared thread-pool barrier-reserve path to gather budget slots. + for (U32 i = 0; i < count; i += 1) { + if (!semaphore_take(semaphore, endt_us)) { + // partial failure: give back what we took so we don't leak permits + semaphore_drop_n(semaphore, i); + return 0; + } + } + return 1; +} + //- rjf: barriers internal Barrier