diff --git a/build.bat b/build.bat index 5c9c71d46..54f70b945 100644 --- a/build.bat +++ b/build.bat @@ -141,7 +141,14 @@ popd pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 if "%raddbg_non_graphical%"=="1" set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1 -if "%radlink%"=="1" set didbuild=1 && %compile% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 +:: NOTE: -DBLAKE3_ATOMICS=1 makes BLAKE3 use C11 _Atomic (plain atomic load) for +:: get_cpu_features instead of MSVC's _InterlockedOr `lock or` barrier on every +:: compress dispatch (was a ~5.5s main-thread hot spot). MSVC C11 atomics require +:: /std:c11 /experimental:c11atomics. Kept external so the vendored blake3 source +:: stays pristine. +set radlink_msvc_flags= +if "%msvc%"=="1" set radlink_msvc_flags=/std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1 +if "%radlink%"=="1" set didbuild=1 && %compile% %radlink_msvc_flags% ..\src\linker\lnk.c %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1 if "%radbin%"=="1" set didbuild=1 && %compile% ..\src\radbin\radbin_main.c %compile_link% %out%radbin.exe || exit /b 1 if "%raddump%"=="1" set didbuild=1 && %compile% ..\src\raddump\raddump_main.c %compile_link% %out%raddump.exe || exit /b 1 if "%ryan_scratch%"=="1" set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c %compile_link% %out%ryan_scratch.exe || exit /b 1 diff --git a/src/base/base_strings.c b/src/base/base_strings.c index 8825d4798..78e3e59ac 100644 --- a/src/base/base_strings.c +++ b/src/base/base_strings.c @@ -213,7 +213,9 @@ str8_cstring_capped(void *cstr, void *cap) { char *ptr = (char *)cstr; char *opl = (char *)cap; - for (;ptr < opl && *ptr != 0; ptr += 1); + // memchr is typically SIMD-accelerated; much faster than a byte-by-byte scan + char *nul = (char *)memchr(ptr, 0, (U64)(opl - ptr)); + ptr = nul ? nul : opl; U64 size = (U64)(ptr - (char *)cstr); String8 result = str8((U8*)cstr, size); return result; diff --git a/src/base/base_threads.h b/src/base/base_threads.h index b34286ccb..9e156d593 100644 --- a/src/base/base_threads.h +++ b/src/base/base_threads.h @@ -129,6 +129,7 @@ internal Semaphore semaphore_open(String8 name); internal void semaphore_close(Semaphore semaphore); internal B32 semaphore_take(Semaphore semaphore, U64 endt_us); internal void semaphore_drop(Semaphore semaphore); +internal void semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall //- rjf: barriers internal Barrier barrier_alloc(U64 count); diff --git a/src/coff/coff_parse.c b/src/coff/coff_parse.c index 1db0b5826..85835b977 100644 --- a/src/coff/coff_parse.c +++ b/src/coff/coff_parse.c @@ -166,11 +166,16 @@ coff_section_header_array_from_name(Arena *arena, String8 string_table, COFF_Sec } +// NOTE: name-skipping variants. coff_read_symbol_name does a cstr scan over the +// memory-mapped string table, which is the dominant cost when parsing symbols in +// bulk; callers that only need the scalar fields (value/section/storage_class/aux, +// e.g. symbol-value interpretation) should use these to avoid that scan. The full +// coff_parse_symbol{16,32} below are these plus the name read, so the scalar-field +// logic lives in exactly one place. internal COFF_ParsedSymbol -coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +coff_parse_symbol32_no_name(COFF_Symbol32 *sym32) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym32->name); result.value = sym32->value; result.section_number = sym32->section_number; result.type = sym32->type; @@ -181,10 +186,9 @@ coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) } internal COFF_ParsedSymbol -coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +coff_parse_symbol16_no_name(COFF_Symbol16 *sym16) { COFF_ParsedSymbol result = {0}; - result.name = coff_read_symbol_name(string_table, &sym16->name); result.value = sym16->value; if (sym16->section_number == COFF_Symbol_DebugSection16) { result.section_number = COFF_Symbol_DebugSection32; @@ -200,6 +204,22 @@ coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) return result; } +internal COFF_ParsedSymbol +coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32) +{ + COFF_ParsedSymbol result = coff_parse_symbol32_no_name(sym32); + result.name = coff_read_symbol_name(string_table, &sym32->name); + return result; +} + +internal COFF_ParsedSymbol +coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16) +{ + COFF_ParsedSymbol result = coff_parse_symbol16_no_name(sym16); + result.name = coff_read_symbol_name(string_table, &sym16->name); + return result; +} + internal COFF_ParsedSymbol coff_parse_symbol(COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx) { diff --git a/src/coff/coff_parse.h b/src/coff/coff_parse.h index edcb29520..028e292f5 100644 --- a/src/coff/coff_parse.h +++ b/src/coff/coff_parse.h @@ -261,6 +261,8 @@ internal String8 coff_name_from_section_header (String8 str internal COFF_ParsedSymbol coff_parse_symbol32(String8 string_table, COFF_Symbol32 *sym32); internal COFF_ParsedSymbol coff_parse_symbol16(String8 string_table, COFF_Symbol16 *sym16); +internal COFF_ParsedSymbol coff_parse_symbol32_no_name(COFF_Symbol32 *sym32); +internal COFF_ParsedSymbol coff_parse_symbol16_no_name(COFF_Symbol16 *sym16); internal COFF_ParsedSymbol coff_parse_symbol (COFF_FileHeaderInfo header, String8 string_table, String8 symbol_table, U32 symbol_idx); internal COFF_Symbol32Array coff_symbol_array_from_data_16(Arena *arena, String8 data, U64 symbol_array_off, U64 symbol_count); diff --git a/src/linker/lnk.c b/src/linker/lnk.c index 250a33c45..0eafb8439 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -1623,7 +1623,8 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) LNK_Symbol *symbol = c->v[i].symbol; LNK_ObjSymbolRef symbol_ref = lnk_ref_from_symbol(symbol); - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // skip the name scan here; the resolved name is already cached on symbol->name + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(symbol_ref.obj, symbol_ref.symbol_idx); COFF_SymbolValueInterpType symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed); if (symbol_interp == COFF_SymbolValueInterp_Undefined) { U32 member_idx; @@ -1641,11 +1642,11 @@ THREAD_POOL_TASK_FUNC(lnk_search_lib_task) if (search_anti_deps) { LNK_ObjSymbolRef dep_symbol = {0}; if (lnk_resolve_weak_symbol(symtab, symbol_ref, &dep_symbol)) { - COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx(dep_symbol.obj, dep_symbol.symbol_idx); + COFF_ParsedSymbol dep_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dep_symbol.obj, dep_symbol.symbol_idx); COFF_SymbolValueInterpType dep_interp = coff_interp_from_parsed_symbol(dep_parsed); if (dep_interp == COFF_SymbolValueInterp_Weak) { U32 member_idx; - if (lnk_search_lib(lib, symbol_parsed.name, &member_idx)) { + if (lnk_search_lib(lib, symbol->name, &member_idx)) { lnk_queue_lib_member(arena, task->imports_hm, task->link->lib_member_infos_hm, member_ref_list, symbol, lib, lib_member_infos, member_idx); } } @@ -1784,6 +1785,7 @@ lnk_link_inputs(TP_Context *tp, if (null_symbol == 0) { null_symbol = push_array(inputer->arena, LNK_Symbol, 1); null_symbol->refs = push_array(inputer->arena, LNK_ObjSymbolRefNode, 1); + null_symbol->refs_tail = null_symbol->refs; null_symbol->refs->v.obj = &link->objs.first->data; } LNK_LibMemberRef *member_refs = push_array(scratch.arena, LNK_LibMemberRef, lib->member_count); @@ -1879,7 +1881,8 @@ lnk_link_inputs(TP_Context *tp, AssertAlways(member_ref->link_symbol->refs != import_stub->refs); // replace the import symbol with a stub, which is later replaced with the real import symbol once import obj is ready. - member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs = import_stub->refs; + member_ref->link_symbol->refs_tail = import_stub->refs_tail; // push import member for import obj generation lnk_lib_member_ref_list_push_node(&link->imports, member_ref); @@ -2950,15 +2953,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_comdat_leaders_task) task->u.patch_symtabs.was_symbol_patched[obj_idx][symbol_idx] = 1; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } } @@ -3022,15 +3018,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_leaders_task) COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol); U64 section_number = task->u.patch_symtabs.common_block_sect->sect_idx + 1; - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = parsed_symbol.raw_symbol; - symbol32->value = contrib->u.offset; - symbol32->section_number = safe_cast_u32(section_number); - } else { - COFF_Symbol16 *symbol16 = parsed_symbol.raw_symbol; - symbol16->value = contrib->u.offset; - symbol16->section_number = safe_cast_u16(section_number); - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = contrib->u.offset; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = safe_cast_u32(section_number); task->u.patch_symtabs.was_symbol_patched[symbol_ref.obj->input_idx][symbol_ref.symbol_idx] = 1; } @@ -3055,17 +3044,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_common_block_symbols_task) COFF_ParsedSymbol defn_parsed = lnk_parsed_from_symbol(defn); Assert(lnk_interp_from_symbol(defn) == COFF_SymbolValueInterp_Regular); if (defn) { - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = defn_parsed.section_number; - symbol32->value = safe_cast_u32(defn_parsed.value); - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(defn_parsed.section_number); - symbol16->value = safe_cast_u32(defn_parsed.value); - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = defn_parsed.section_number; + obj->parsed_symbols[symbol_idx].value = defn_parsed.value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } } } @@ -3103,15 +3084,8 @@ THREAD_POOL_TASK_FUNC(lnk_patch_regular_symbols_task) value = sc->u.off + symbol.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(section_number); - symbol16->value = value; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; } } ProfEnd(); @@ -3148,17 +3122,9 @@ lnk_patch_obj_symtab(LNK_SymbolTable *symtab, LNK_Obj *obj, B8 *was_symbol_patch value = fixup_src.value; } - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = fixup_dst.raw_symbol; - symbol32->section_number = section_number; - symbol32->value = value; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = fixup_dst.raw_symbol; - symbol16->section_number = (U16)section_number; - symbol16->value = value; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = section_number; + obj->parsed_symbols[symbol_idx].value = value; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; was_symbol_patched[symbol_idx] = 1; } @@ -3839,17 +3805,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) if (sect && (~sect->flags & COFF_SectionFlag_LnkRemove)) { if (~sect->flags & COFF_SectionFlag_MemDiscardable) { LNK_SectionContrib *first_sc = lnk_get_first_section_contrib(sect); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = safe_cast_u32(first_sc->u.sect_idx + 1); - symbol32->value = first_sc->u.off; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(first_sc->u.sect_idx + 1); - symbol16->value = first_sc->u.off; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = safe_cast_u32(first_sc->u.sect_idx + 1); + obj->parsed_symbols[symbol_idx].value = first_sc->u.off; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; } else { lnk_error_obj(LNK_Error_SectRefsDiscardedMemory, obj, "symbol %S (No. 0x%llx) references section with discard flag", symbol.name, symbol_idx); } @@ -3867,17 +3825,9 @@ THREAD_POOL_TASK_FUNC(lnk_patch_section_symbols_task) LNK_Section *fallback_sect = task->image_sects.v[task->image_sects.count-1]; U32 fallback_section_number = safe_cast_u32(fallback_sect->sect_idx + 1); U32 fallback_section_offset = safe_cast_u32(fallback_voff - fallback_sect->voff); - if (obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol.raw_symbol; - symbol32->section_number = fallback_section_number; - symbol32->value = fallback_section_offset; - symbol32->storage_class = COFF_SymStorageClass_Static; - } else { - COFF_Symbol16 *symbol16 = symbol.raw_symbol; - symbol16->section_number = safe_cast_u16(fallback_section_number); - symbol16->value = fallback_section_offset; - symbol16->storage_class = COFF_SymStorageClass_Static; - } + obj->parsed_symbols[symbol_idx].section_number = fallback_section_number; + obj->parsed_symbols[symbol_idx].value = fallback_section_offset; + obj->parsed_symbols[symbol_idx].storage_class = COFF_SymStorageClass_Static; lnk_error_obj(LNK_Warning_UndefinedSectionSymbol, obj, "undefined section symbol %S (No. 0x%llx) refers to an image section that doesn't exist; patching to %#llx", symbol.name, symbol_idx, fallback_voff); } @@ -4743,7 +4693,11 @@ lnk_build_image(TP_Arena *arena, TP_Context *tp, LNK_Config *config, LNK_SymbolT ProfBeginV("Alloc Image Buffer [%M]", lnk_section_table_total_fsize(sectab)); image_data.size = lnk_section_table_total_fsize(sectab) + image_string_table.total_size; - image_data.str = push_array_no_zero(arena->v[0], U8, image_data.size); + // Standalone reservation (not the shared link arena) so it can be released the instant the image + // is written to disk -- VirtualFree returns fast and the kernel zeroes this ~1GB on its background + // thread, overlapping the rest of the run, instead of in the single-threaded exit rundown. + image_data.str = reserve_memory(image_data.size); + commit_memory(image_data.str, image_data.size); ProfEnd(); ProfBegin("Fill Align Bytes"); @@ -5262,6 +5216,7 @@ lnk_write_thread(void *raw_ctx) ProfEnd(); } + internal void lnk_log_timers(void) { @@ -5350,6 +5305,52 @@ lnk_debug_filter_objs(Arena *arena, LNK_Obj **objs, U64 objs_count, U64 *count_o return debug_info_objs; } +// Parallel release of memory-mapped input file views. +// Inputs are mapped copy-on-write (PAGE_WRITECOPY/FILE_MAP_COPY); pages touched +// during linking become private-dirty and are reclaimed by the kernel in +// single-threaded process rundown at exit (~3s for a large link). Unmapping them +// in parallel before exit moves that reclaim off the serial post-exit path. +typedef struct LNK_UnmapViewTask +{ + String8 *views; +} LNK_UnmapViewTask; + +internal +THREAD_POOL_TASK_FUNC(lnk_unmap_view_task) +{ + LNK_UnmapViewTask *task = raw_task; + String8 view = task->views[task_id]; +#if OS_WINDOWS + UnmapViewOfFile(view.str); +#elif OS_LINUX + munmap(view.str, view.size); +#endif +} + +internal void +lnk_release_input_views(TP_Context *tp, LNK_Inputer *inputer) +{ + Temp scratch = scratch_begin(0, 0); + + // collect distinct whole-file mapped views (is_thin); skip lib-member + // substrings and linkgen arena data + U64 cap = inputer->objs.count + inputer->libs.count; + String8 *views = push_array_no_zero(scratch.arena, String8, cap); + U64 count = 0; + for EachNode(n, LNK_Input, inputer->objs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + for EachNode(n, LNK_Input, inputer->libs.first) { if (n->is_thin && n->data.size) { views[count++] = n->data; } } + + if (count > 0) { + U64 begin_us = now_time_us(); + LNK_UnmapViewTask task = { .views = views }; + tp_for_parallel(tp, 0, count, lnk_unmap_view_task, &task); + U64 end_us = now_time_us(); + lnk_log(LNK_Log_Timers, "Released %llu input views in %.2f ms", count, (F64)(end_us - begin_us) / 1000.0); + } + + scratch_end(scratch); +} + internal void lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) { @@ -5368,7 +5369,7 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) LNK_SymbolTable *symtab = lnk_symbol_table_init(arena); // - // Link Image + // Link Image (group digests, if any, are synthesized + consumed inside lnk_link_image) // LNK_LinkResult link = lnk_link_image(tp, arena, config, inputer, symtab); @@ -5545,6 +5546,20 @@ lnk_run_linker(TP_Context *tp, TP_Arena *arena, LNK_Config *config) // wait for the thread to finish writing image to disk thread_join(image_write_thread, -1); + // image is on disk and no longer read by anyone -- release its ~1GB now so the kernel reclaims it + // concurrently with the remaining work + exit, not single-threaded in the process rundown. + release_memory(image_ctx.image_data.str, image_ctx.image_data.size); + + // outputs are written and inputs are no longer read; release the copy-on-write + // input views in parallel so their dirty pages are reclaimed here (multi-threaded) + // instead of in single-threaded process rundown at exit. Only safe for the CoW + // (read-only) mapping mode; read-write-shared would flush dirty pages back to the + // input files on unmap. + if ((config->io_flags & LNK_IO_Flags_MemoryMapFilesReadOnly) && + !(config->io_flags & LNK_IO_Flags_MemoryMapFilesReadWrite)) { + lnk_release_input_views(tp, inputer); + } + // // Timers // @@ -5839,8 +5854,8 @@ entry_point(CmdLine *cmdline) } switch (config->boot_mode) { - case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; - case LNK_BootMode_TypeServer: lnk_run_type_server(tp, tp_arena, config); break; + case LNK_BootMode_Linker: lnk_run_linker (tp, tp_arena, config); break; + case LNK_BootMode_TypeServer: lnk_run_type_server (tp, tp_arena, config); break; } lnk_log_end(); diff --git a/src/linker/lnk_debug_info.c b/src/linker/lnk_debug_info.c index 83f574969..87aad99d9 100644 --- a/src/linker/lnk_debug_info.c +++ b/src/linker/lnk_debug_info.c @@ -1374,29 +1374,23 @@ lnk_hash_cv_leaf_deep(Arena *arena, temp_end(temp); } -internal LNK_LeafRef * -lnk_leaf_hash_table_search(LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) +// Returns the type index assigned to leaf_ref's deduped class, or 0 if absent. Dedup is by hash +// (lnk_match_leaf_ref is a_hash==b_hash), so the assigned-ti table is keyed purely by hash -- a single +// probe, deref-free (the occupant hash is on the slot), and sized to the UNIQUE leaf count. +internal CV_TypeIndex +lnk_leaf_hash_table_search_ti(LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref) { - LNK_LeafRef *match = 0; - - CV_DebugT *debug_t = &input->debug_t_arr[leaf_ref.obj_idx]; - CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; - U64 hash = debug_h->v[leaf_ref.leaf_idx]; - U64 best_bucket_idx = hash % ht->cap; - U64 bucket_idx = best_bucket_idx; + CV_DebugH *debug_h = &input->debug_h_arr[leaf_ref.obj_idx]; + U64 hash = debug_h->v[leaf_ref.leaf_idx]; + U64 best_idx = hash % ht->cap; + U64 idx = best_idx; do { - LNK_LeafRef *bucket = ht->bucket_arr[bucket_idx]; - if (bucket == 0) { break; } - - if (lnk_match_leaf_ref(input, *bucket, leaf_ref)) { - match = bucket; - break; - } + if (ht->ti_arr[idx] == 0) { break; } // empty slot -> not present + if (ht->hash_arr[idx] == hash) { return ht->ti_arr[idx]; } + idx = (idx + 1) == ht->cap ? 0 : (idx + 1); + } while (idx != best_idx); - bucket_idx = (bucket_idx + 1) == ht->cap ? 0 : (bucket_idx + 1); - } while (bucket_idx != best_bucket_idx); - - return match; + return 0; } internal @@ -1748,57 +1742,39 @@ THREAD_POOL_TASK_FUNC(lnk_assign_type_indices_task) { LNK_MergeTypes *task = raw_task; - CV_TypeIndexSource ti_source = task->ti_source; - LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; - CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; - U64 assigned_type_cap = task->assigned_type_caps[ti_source]; - CV_TypeIndex *assigned_type_ht = task->assigned_type_hts[ti_source]; + CV_TypeIndexSource ti_source = task->ti_source; + LNK_LeafRefArray unique_leaf_refs = task->unique_leaf_refs_arr[ti_source]; + CV_TypeIndex min_type_index = task->min_type_indices[ti_source]; + LNK_AssignedTiHash *at = &task->assigned_ti_arr[ti_source]; + CV_DebugH *debug_h_arr = task->input->debug_h_arr; + // Insert each unique leaf's assigned type index into the (unique-sized) hash->ti table, keyed by leaf + // hash. Unique leaves have distinct hashes (dedup is by hash), so each claims its own empty slot. + // search_ti (in the later fixup phase, after this barrier) recovers ti in one deref-free probe. for EachInRange(i, task->ranges[task_id]) { LNK_LeafRef *leaf_ref = unique_leaf_refs.v[i]; CV_TypeIndex type_index = min_type_index + i; - U64 hash = u64_hash_from_str8(str8_struct(leaf_ref)); - U64 best_idx = hash % assigned_type_cap; + U64 hash = debug_h_arr[leaf_ref->obj_idx].v[leaf_ref->leaf_idx]; + U64 best_idx = hash % at->cap; U64 idx = best_idx; - B32 is_inserted = 0; + B32 is_assigned = 0; do { - CV_TypeIndex curr_type_index = assigned_type_ht[idx]; - if (curr_type_index == 0) { - CV_TypeIndex cmp_type_index = ins_atomic_u32_eval_cond_assign(&assigned_type_ht[idx], type_index, curr_type_index); - if (cmp_type_index == curr_type_index) { - is_inserted = 1; + if (at->ti_arr[idx] == 0) { + CV_TypeIndex cmp = ins_atomic_u32_eval_cond_assign(&at->ti_arr[idx], type_index, 0); + if (cmp == 0) { + at->hash_arr[idx] = hash; // only this worker owns the slot now; read back in the fixup phase + is_assigned = 1; break; } } - // advance - idx = (idx + 1) == assigned_type_cap ? 0 : (idx + 1); + idx = (idx + 1) == at->cap ? 0 : (idx + 1); } while (idx != best_idx); - Assert(is_inserted); + Assert(is_assigned); } } -internal CV_TypeIndex -lnk_assigned_type_ht_search(U64 cap, CV_TypeIndex *ht, CV_TypeIndex min_type_index, LNK_LeafRefArray unique_leaf_refs, LNK_LeafRef *v, U64 hash) -{ - U64 best_idx = hash % cap; - U64 idx = best_idx; - do { - CV_TypeIndex type_index = ht[idx]; - if (type_index < min_type_index) { break; } - - U64 leaf_idx = type_index - min_type_index; - LNK_LeafRef *compar = unique_leaf_refs.v[leaf_idx]; - if (MemoryMatchStruct(compar,v)) { return type_index; } - - idx = (idx + 1) == cap ? 0 : (idx + 1); - } while(idx != best_idx); - - InvalidPath; - return 0; -} - internal void lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_TypeIndexInfoList ti_info_list) { @@ -1809,21 +1785,11 @@ lnk_fixup_cv_type_indices(LNK_MergeTypes *ctx, U32 obj_idx, String8 data, CV_Typ // skip basic types if (ti < ctx->input->min_type_indices[n->source]) { continue; } - CV_TypeIndex final_ti = 0; - LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); - LNK_LeafHashTable *leaf_ht = &ctx->leaf_ht_arr[n->source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, ctx->input, leaf_ref); - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - final_ti = lnk_assigned_type_ht_search(ctx->assigned_type_caps [n->source], - ctx->assigned_type_hts [n->source], - ctx->min_type_indices [n->source], - ctx->unique_leaf_refs_arr[n->source], - final_leaf, - final_hash); - } + LNK_LeafRef leaf_ref = lnk_leaf_ref_from_ti(ctx->input, obj_idx, n->source, ti); + LNK_AssignedTiHash *assigned = &ctx->assigned_ti_arr[n->source]; + CV_TypeIndex final_ti = lnk_leaf_hash_table_search_ti(assigned, ctx->input, leaf_ref); #if BUILD_DEBUG - else { + if (final_ti == 0) { lnk_error_obj(LNK_Error_InvalidTypeIndex, ctx->input->obj_arr[obj_idx], "no itype 0x%x", ti); } #endif @@ -1992,23 +1958,9 @@ THREAD_POOL_TASK_FUNC(lnk_build_obj_ti_map) for EachIndex(leaf_idx, debug_t->count) { CV_Leaf leaf = cv_debug_t_get_leaf(debug_t, leaf_idx); CV_TypeIndexSource source = cv_type_index_source_from_leaf_kind(leaf.kind); - LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; - LNK_LeafHashTable *leaf_ht = &task->leaf_ht_arr[source]; - LNK_LeafRef *final_leaf = lnk_leaf_hash_table_search(leaf_ht, input, leaf_ref); - - if (final_leaf) { - U64 final_hash = u64_hash_from_str8(str8_struct(final_leaf)); - CV_TypeIndex final_ti = lnk_assigned_type_ht_search(task->assigned_type_caps [source], - task->assigned_type_hts [source], - task->min_type_indices [source], - task->unique_leaf_refs_arr[source], - final_leaf, - final_hash); - - obj_ti_map[leaf_idx] = final_ti; - } else { - obj_ti_map[leaf_idx] = 0; - } + LNK_LeafRef leaf_ref = { obj_idx, leaf_idx }; + LNK_AssignedTiHash *assigned = &task->assigned_ti_arr[source]; + obj_ti_map[leaf_idx] = lnk_leaf_hash_table_search_ti(assigned, input, leaf_ref); } task->result.obj_ti_maps[obj_idx] = obj_ti_map; @@ -2152,6 +2104,12 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK task.unique_leaf_refs_arr[ti_source].count = sum_array_u64(tp->worker_count, task.counts[ti_source]); task.unique_leaf_refs_arr[ti_source].v = push_array_no_zero(scratch.arena, LNK_LeafRef *, task.unique_leaf_refs_arr[ti_source].count); + + // assigned-ti table sized to the unique (deduped) count -- not the total leaf count, which would + // add the bucket-parallel ti/hash arrays' worth of peak working set (~3GB on large links) + task.assigned_ti_arr[ti_source].cap = 1 + ((task.unique_leaf_refs_arr[ti_source].count * 13) / 10); // * 1.3 + task.assigned_ti_arr[ti_source].ti_arr = push_array(scratch.arena, CV_TypeIndex, task.assigned_ti_arr[ti_source].cap); + task.assigned_ti_arr[ti_source].hash_arr = push_array(scratch.arena, U64, task.assigned_ti_arr[ti_source].cap); task.offsets[ti_source] = offsets_from_counts_array_u64(scratch.arena, task.counts[ti_source], tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_get_present_buckets_task, &task, "Copy present buckets"); @@ -2250,8 +2208,6 @@ lnk_merge_types(TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK ProfBegin("Assign type indices"); for EachIndex(ti_source, CV_TypeIndexSource_COUNT) { task.ti_source = ti_source; - task.assigned_type_caps[ti_source] = (task.unique_leaf_refs_arr[ti_source].count * 13) / 10; - task.assigned_type_hts [ti_source] = push_array(scratch.arena, CV_TypeIndex, task.assigned_type_caps[ti_source]); task.min_type_indices [ti_source] = CV_MinComplexTypeIndex; task.ranges = tp_divide_work(scratch.arena, task.unique_leaf_refs_arr[ti_source].count, tp->worker_count); tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_assign_type_indices_task, &task, "Assign Type Indices"); diff --git a/src/linker/lnk_debug_info.h b/src/linker/lnk_debug_info.h index dc094e0f9..aa49eee42 100644 --- a/src/linker/lnk_debug_info.h +++ b/src/linker/lnk_debug_info.h @@ -115,10 +115,20 @@ typedef struct { U64 count; LNK_LeafRef **v; } LNK_LeafRefArray; typedef struct { - U64 cap; + U64 cap; // ~1.3x total (pre-dedup) leaf count LNK_LeafRef **bucket_arr; } LNK_LeafHashTable; +// Maps leaf hash -> assigned type index, sized to the UNIQUE (post-dedup) leaf count rather than the +// total leaf count -- folds the canonical-bucket + bucket->ti lookups into one probe (deref-free: the +// hash is stored on the slot) without the total-sized per-bucket arrays that would add ~3GB to peak. +typedef struct +{ + U64 cap; // ~1.3x unique leaf count + CV_TypeIndex *ti_arr; // assigned type index per slot; 0 == empty (ti is always >= CV_MinComplexTypeIndex) + U64 *hash_arr; // occupant leaf hash (parallel to ti_arr); disambiguates open-addressing collisions +} LNK_AssignedTiHash; + typedef struct LNK_LeafRange { struct LNK_LeafRange *next; @@ -150,6 +160,7 @@ typedef struct LNK_CodeViewInput *input; CV_DebugS *debug_s_arr; LNK_LeafHashTable leaf_ht_arr[CV_TypeIndexSource_COUNT]; + LNK_AssignedTiHash assigned_ti_arr[CV_TypeIndexSource_COUNT]; Arena **fixed_arenas; CV_TypeIndexSource ti_source; U32Array indices; @@ -174,8 +185,6 @@ typedef struct U64 pass_idx; // assign type indices - U64 assigned_type_caps [CV_TypeIndexSource_COUNT]; - CV_TypeIndex *assigned_type_hts [CV_TypeIndexSource_COUNT]; CV_TypeIndex min_type_indices [CV_TypeIndexSource_COUNT]; LNK_LeafRefArray unique_leaf_refs_arr[CV_TypeIndexSource_COUNT]; @@ -251,7 +260,7 @@ internal B32 lnk_match_leaf_ref (LNK_CodeViewInput internal U64 lnk_hash_cv_leaf (LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list, B32 discard_cycles); internal void lnk_hash_cv_leaf_deep (Arena *arena, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref, CV_TypeIndexInfoList ti_info_list); internal LNK_LeafRef * lnk_leaf_hash_table_insert_or_update(LNK_LeafHashTable *leaf_ht, LNK_CodeViewInput *input, CV_DebugH *hashes, U64 hash, LNK_LeafRef *new_bucket); -internal LNK_LeafRef * lnk_leaf_hash_table_search (LNK_LeafHashTable *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); +internal CV_TypeIndex lnk_leaf_hash_table_search_ti (LNK_AssignedTiHash *ht, LNK_CodeViewInput *input, LNK_LeafRef leaf_ref); // returns assigned ti for leaf_ref's hash class, 0 if absent internal LNK_MergedTypes lnk_merge_types (TP_Context *tp, TP_Arena *tp_temp, LNK_CodeViewInput *input, LNK_MergeTypeFlags merge_flags); internal void lnk_replace_type_names_with_hashes (TP_Context *tp, TP_Arena *arena, U64 leaf_count, U8 **leaf_arr, LNK_TypeNameHashMode mode, U64 hash_length, String8 map_name); diff --git a/src/linker/lnk_obj.c b/src/linker/lnk_obj.c index dabd77f83..fcc541c5e 100644 --- a/src/linker/lnk_obj.c +++ b/src/linker/lnk_obj.c @@ -125,13 +125,16 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) } // - // error check symbol table + // error check symbol table (+ memoize parsed symbols) // + LNK_ParsedSymbolLite *parsed_symbols = push_array(arena, LNK_ParsedSymbolLite, header.symbol_count); { COFF_SectionHeader *section_table = (COFF_SectionHeader *)str8_substr(input->data, header.section_table_range).str; COFF_ParsedSymbol symbol; for (U64 symbol_idx = 0; symbol_idx < header.symbol_count; symbol_idx += (1 + symbol.aux_symbol_count)) { symbol = coff_parse_symbol(header, raw_coff_string_table, raw_coff_symbol_table, symbol_idx); + U32 raw_off = symbol.raw_symbol ? safe_cast_u32((U8 *)symbol.raw_symbol - input->data.str) : 0; + parsed_symbols[symbol_idx] = (LNK_ParsedSymbolLite){ raw_off, safe_cast_u32(symbol.value), symbol.section_number, symbol.type, symbol.storage_class, symbol.aux_symbol_count }; COFF_SymbolValueInterpType interp = coff_interp_symbol(symbol.section_number, symbol.value, symbol.storage_class); if (interp == COFF_SymbolValueInterp_Regular) { if (symbol.section_number == 0 || symbol.section_number > header.section_count_no_null) { @@ -335,6 +338,7 @@ THREAD_POOL_TASK_FUNC(lnk_obj_initer) obj->path = push_str8_copy(arena, input->path); obj->header = header; obj->section_flags = section_flags; + obj->parsed_symbols = parsed_symbols; obj->comdats = comdats; obj->exclude_from_debug_info = input->exclude_from_debug_info; obj->hotpatch = hotpatch; @@ -570,8 +574,11 @@ lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx) return sect_idx+1; } +// NOTE: skips section.name (coff_name_from_section_header does a string-table +// lookup); use when only the header/flags/ranges are needed (e.g. checking +// section flags). Callers that need the name should use lnk_obj_section_from_sect_idx. internal LNK_ObjSection -lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx) { Assert(sect_idx < obj->header.section_count_no_null); LNK_ObjSection section = {0}; @@ -580,13 +587,20 @@ lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) section.section_number = sect_idx+1; section.header = &lnk_coff_section_table_from_obj(obj)[sect_idx]; section.flags = &obj->section_flags[sect_idx]; - section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); section.vrange = rng_1u64(section.header->voff, section.header->voff + section.header->vsize); section.frange = rng_1u64(section.header->foff, section.header->foff + section.header->fsize); section.reloc_count = section.header->reloc_count; return section; } +internal LNK_ObjSection +lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx) +{ + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); + return section; +} + internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number) { @@ -647,19 +661,35 @@ lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number) return §ion_table[sect_idx]; } +// NOTE: returns the memoized parse built in lnk_obj_initer. The struct is mutable: symbol-value +// patching (lnk_patch_*_task) writes value/section_number/storage_class here, NOT into the mmapped +// obj->data symbol table -- so obj->data symbol values are never written after load. raw_symbol still +// points into obj->data (read-only) for aux-record reads (coff_parse_weak_tag / coff_parse_secdef). internal COFF_ParsedSymbol -lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) +lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx) { - String8 string_table = str8_substr(obj->data, obj->header.string_table_range); - String8 symbol_table = str8_substr(obj->data, obj->header.symbol_table_range); - + LNK_ParsedSymbolLite *lite = &obj->parsed_symbols[symbol_idx]; COFF_ParsedSymbol result = {0}; - if (obj->header.is_big_obj) { - result = coff_parse_symbol32(string_table, (COFF_Symbol32 *)symbol_table.str + symbol_idx); - } else { - result = coff_parse_symbol16(string_table, (COFF_Symbol16 *)symbol_table.str + symbol_idx); + result.value = lite->value; + result.section_number = lite->section_number; + result.type = lite->type; + result.storage_class = lite->storage_class; + result.aux_symbol_count = lite->aux_symbol_count; + result.raw_symbol = lite->raw_symbol_off ? (obj->data.str + lite->raw_symbol_off) : 0; // offset -> ptr + return result; +} + +internal COFF_ParsedSymbol +lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx) +{ + COFF_ParsedSymbol result = lnk_parsed_symbol_from_coff_symbol_idx_no_name(obj, symbol_idx); + // name is excluded from the memo -- decode it from the (read-only) symbol record on demand. Patching + // never touches the name, so re-deriving from raw_symbol stays correct after value/section patches. + if (result.raw_symbol) { + String8 string_table = str8_substr(obj->data, obj->header.string_table_range); + if (obj->header.is_big_obj) { result.name = coff_parse_symbol32(string_table, (COFF_Symbol32 *)result.raw_symbol).name; } + else { result.name = coff_parse_symbol16(string_table, (COFF_Symbol16 *)result.raw_symbol).name; } } - return result; } @@ -756,8 +786,10 @@ lnk_raw_directives_from_obj(Arena *arena, LNK_Obj *obj) { String8List drectve_data = {0}; for (U64 sect_idx = 0; sect_idx < obj->header.section_count_no_null; sect_idx += 1) { - LNK_ObjSection section = lnk_obj_section_from_sect_idx(obj, sect_idx); + // only LnkInfo sections (rare) need the name; skip the string-table lookup for the rest + LNK_ObjSection section = lnk_obj_section_from_sect_idx_no_name(obj, sect_idx); if (*section.flags & COFF_SectionFlag_LnkInfo) { + section.name = coff_name_from_section_header(lnk_coff_string_table_from_obj(obj), section.header); if (str8_match(section.name, str8_lit(".drectve"), 0)) { if (*section.flags & COFF_SectionFlag_CntUninitializedData) { lnk_error_obj(LNK_Error_IllData, obj, ".drectve section header has flag COFF_SectionFlag_CntUninitializedData"); diff --git a/src/linker/lnk_obj.h b/src/linker/lnk_obj.h index d13a3d1e1..95df7bde7 100644 --- a/src/linker/lnk_obj.h +++ b/src/linker/lnk_obj.h @@ -5,6 +5,21 @@ // --- Input ------------------------------------------------------------------- +// Slim memoized symbol parse: every COFF_ParsedSymbol field EXCEPT name (the 16B String8). Sized by +// total symbol count, so dropping name saves ~16B/sym of peak. The name is the cold path -- it is +// re-decoded from raw_symbol on demand in lnk_parsed_symbol_from_coff_symbol_idx (the named accessor); +// the hot _no_name accessor and all symbol-value patching never touch it. +typedef struct LNK_ParsedSymbolLite +{ + U32 raw_symbol_off; // byte offset of the COFF symbol record within obj->data (0 = none). + // stored as an offset, not a pointer, to keep this struct 16B. + U32 value; // COFF symbol value is U32 (section-relative offset / size / etc.) + U32 section_number; + COFF_SymbolType type; // U16 + COFF_SymStorageClass storage_class; // U8 + U8 aux_symbol_count; +} LNK_ParsedSymbolLite; + typedef struct LNK_Obj { String8 path; @@ -12,6 +27,8 @@ typedef struct LNK_Obj COFF_FileHeaderInfo header; COFF_SectionFlags *section_flags; + LNK_ParsedSymbolLite *parsed_symbols; // memoized parse per symbol_idx (aux slots zeroed), name excluded. + // Mutable: symbol-value patching writes here, NOT obj->data. // flags B8 hotpatch; @@ -146,9 +163,11 @@ internal LNK_Symbol * lnk_obj_get_comdat_symlink(LNK_Obj *obj, U64 section_n internal COFF_SectionHeader * lnk_coff_section_header_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx(LNK_Obj *obj, U64 symbol_idx); +internal COFF_ParsedSymbol lnk_parsed_symbol_from_coff_symbol_idx_no_name(LNK_Obj *obj, U64 symbol_idx); internal U64 lnk_obj_sect_idx_from_section_number(LNK_Obj *obj, U64 section_number); internal U64 lnk_obj_section_number_from_sect_idx(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_sect_idx(LNK_Obj *obj, U64 sect_idx); +internal LNK_ObjSection lnk_obj_section_from_sect_idx_no_name(LNK_Obj *obj, U64 sect_idx); internal LNK_ObjSection lnk_obj_section_from_section_number(LNK_Obj *obj, U64 section_number); internal COFF_RelocArray lnk_coff_relocs_from_section_header(LNK_Obj *obj, COFF_SectionHeader *section_header); internal String8 lnk_coff_string_table_from_obj(LNK_Obj *obj); diff --git a/src/linker/lnk_symbol_table.c b/src/linker/lnk_symbol_table.c index 6cd8734ef..d1d117d08 100644 --- a/src/linker/lnk_symbol_table.c +++ b/src/linker/lnk_symbol_table.c @@ -11,6 +11,7 @@ lnk_make_symbol(Arena *arena, String8 name, LNK_Obj *obj, U32 symbol_idx) LNK_Symbol *symbol = push_array(arena, LNK_Symbol, 1); symbol->name = name; symbol->refs = ref; + symbol->refs_tail = ref; return symbol; } @@ -105,12 +106,14 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) { B32 can_replace = 0; - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_ParsedSymbol src_parsed = lnk_parsed_from_symbol(src); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); - COFF_SymbolValueInterpType src_interp = lnk_interp_from_symbol(src); + // only scalar fields + raw_symbol are needed here (names come from dst->name/src->name); + // skip the symbol-name string-table scan and avoid re-parsing for interp LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); LNK_ObjSymbolRef src_ref = lnk_ref_from_symbol(src); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_ParsedSymbol src_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(src_ref.obj, src_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); + COFF_SymbolValueInterpType src_interp = coff_interp_from_parsed_symbol(src_parsed); LNK_Obj *dst_obj = dst_ref.obj; LNK_Obj *src_obj = src_ref.obj; @@ -332,9 +335,10 @@ lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src) internal void lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) { - COFF_ParsedSymbol dst_parsed = lnk_parsed_from_symbol(dst); - COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst); + // only scalar fields are needed below, so skip the symbol-name string-table scan LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst); + COFF_ParsedSymbol dst_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(dst_ref.obj, dst_ref.symbol_idx); + COFF_SymbolValueInterpType dst_interp = coff_interp_from_parsed_symbol(dst_parsed); if (dst_interp == COFF_SymbolValueInterp_Regular) { // remove replaced section from the output @@ -350,10 +354,9 @@ lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src) } } - // merge symbol refs - LNK_ObjSymbolRefNode *src_last_ref; - for (src_last_ref = src->refs; src_last_ref->next != 0; src_last_ref = src_last_ref->next); - src_last_ref->next = dst->refs; + // merge symbol refs (append dst's list onto src's tail; tail pointer keeps this O(1)) + src->refs_tail->next = dst->refs; + src->refs_tail = dst->refs_tail; // assert leader section is live #if BUILD_DEBUG @@ -532,7 +535,9 @@ lnk_parsed_from_symbol(LNK_Symbol *symbol) internal COFF_SymbolValueInterpType lnk_interp_from_symbol(LNK_Symbol *symbol) { - COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol); + // interp only needs scalar fields; skip the symbol-name string-table scan + LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol); + COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx_no_name(ref.obj, ref.symbol_idx); return coff_interp_from_parsed_symbol(symbol_parsed); } @@ -773,17 +778,9 @@ THREAD_POOL_TASK_FUNC(lnk_replace_weak_with_default_symbol_task) COFF_SymbolValueInterpType resolve_interp = coff_interp_from_parsed_symbol(resolve_parsed); if (resolve_interp == COFF_SymbolValueInterp_Weak) { COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(resolve_parsed, symbol_ref.obj->header.is_big_obj); - if (symbol_ref.obj->header.is_big_obj) { - COFF_Symbol32 *symbol32 = symbol_parsed.raw_symbol; - symbol32->section_number = COFF_Symbol_UndefinedSection; - symbol32->value = 0; - symbol32->storage_class = COFF_SymStorageClass_External; - } else { - COFF_Symbol16 *symbol16 = symbol_parsed.raw_symbol; - symbol16->section_number = COFF_Symbol_UndefinedSection; - symbol16->value = 0; - symbol16->storage_class = COFF_SymStorageClass_External; - } + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].section_number = COFF_Symbol_UndefinedSection; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].value = 0; + symbol_ref.obj->parsed_symbols[symbol_ref.symbol_idx].storage_class = COFF_SymStorageClass_External; } else { symbol->refs->v = resolve; } diff --git a/src/linker/lnk_symbol_table.h b/src/linker/lnk_symbol_table.h index cdf383045..b36a2ba64 100644 --- a/src/linker/lnk_symbol_table.h +++ b/src/linker/lnk_symbol_table.h @@ -21,6 +21,7 @@ typedef struct LNK_Symbol { String8 name; LNK_ObjSymbolRefNode *refs; + LNK_ObjSymbolRefNode *refs_tail; // tail of `refs`, so symbol-ref merges are O(1) instead of O(n) tail walks } LNK_Symbol; // --- Symbol Containers ------------------------------------------------------- diff --git a/src/linker/thread_pool/thread_pool.c b/src/linker/thread_pool/thread_pool.c index 71003bb98..180edb169 100644 --- a/src/linker/thread_pool/thread_pool.c +++ b/src/linker/thread_pool/thread_pool.c @@ -68,12 +68,18 @@ tp_alloc(Arena *arena, U32 worker_count, U32 max_worker_count, String8 name) Semaphore exec_semaphore = {0}; if (worker_count > 1) { main_semaphore = semaphore_alloc(0, 1, str8_zero()); + // Max counts carry 2x headroom: tp_for_parallel wakes workers with a single + // batched ReleaseSemaphore(drop_count); a batch can land while up to + // worker_count-1 previously-woken workers have not yet re-taken their permit, + // so the count can transiently reach ~2*worker_count. A tight max (== worker + // count) would make that batched release exceed the max and fail outright, + // waking no one and deadlocking at the next barrier. if (is_shared) { AssertAlways(worker_count <= max_worker_count); - task_semaphore = semaphore_alloc(0, max_worker_count, name); - exec_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * max_worker_count, name); + exec_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } else { - task_semaphore = semaphore_alloc(0, worker_count, str8_zero()); + task_semaphore = semaphore_alloc(0, 2 * worker_count, str8_zero()); } } @@ -210,18 +216,17 @@ tp_for_parallel(TP_Context *pool, TP_Arena *task_arena, U64 task_count, TP_TaskF U64 drop_count = Min(task_count, pool->worker_count); - // if we are in shared mode ping local semaphore + // Wake exactly drop_count workers in a single batched ReleaseSemaphore. The + // count MUST be drop_count (not drop_count-1): tasks that nest tp_broadcast_ + // (e.g. lnk_walk_relocs_and_mark_ref_sections_task) barrier across all + // worker_count participants, and under-waking by one leaves the barrier short + // -> deadlock. Overflow from the batch is prevented by the 2x semaphore max + // headroom set in tp_alloc. if (pool->exec_semaphore.u64[0] != 0) { - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx +=1) { - semaphore_drop(pool->exec_semaphore); - } - } - - // ping shared semaphore - for (U64 worker_idx = 0; worker_idx < drop_count; worker_idx += 1) { - semaphore_drop(pool->task_semaphore); + semaphore_drop_n(pool->exec_semaphore, (U32)drop_count); } - + semaphore_drop_n(pool->task_semaphore, (U32)drop_count); + // run tasks on main worker tp_run_tasks(pool, &pool->worker_arr[0]); diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index d96751388..bcbe6cc04 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -707,6 +707,19 @@ semaphore_drop(Semaphore semaphore) } } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if(semaphore.u64[0] != 0) + { + for(U32 i = 0; i < count; i += 1) + { + int err = LNX_RETRY_ON_EINTR(sem_post((sem_t*)semaphore.u64[0])); + Assert(err == 0); + } + } +} + //- rjf: barriers internal Barrier diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index 799452038..35db175f1 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -692,6 +692,15 @@ semaphore_drop(Semaphore semaphore) ReleaseSemaphore(handle, 1, 0); } +internal void +semaphore_drop_n(Semaphore semaphore, U32 count) +{ + if (count > 0) { + HANDLE handle = (HANDLE)semaphore.u64[0]; + ReleaseSemaphore(handle, count, 0); + } +} + //- rjf: barriers internal Barrier