Merge branch 'dev' of https://github.com/EpicGames/raddebugger into dev

Author: ryanfleury

.github/workflows/builds.yml

@@ -28,3 +28,5 @@ jobs:
           call build rdi_from_pdb clang debug       || exit /b 1
           call build rdi_from_dwarf clang debug     || exit /b 1
           call build rdi_dump clang debug           || exit /b 1
+          call build radlink msvc debug             || exit /b 1
+          call build radlink clang debug            || exit /b 1

README.md

@@ -158,6 +158,45 @@ like remote debugging, porting to different architectures, further improving
 the debugger's features (like improving the visualization engine), and so on.
 But for now, we're mostly focused on those first two phases.
 
+---
+
+# The RAD Linker
+
+The RAD Linker is a new performance linker for generating x64 PE/COFF binaries. It is designed to be very fast when creating gigantic executables. It generates standard PDB files for debugging, but it can also optionally create RAD Debugger debug info too (useful for huge executables that otherwise create broken PDBs that overflow internal 32-bit tables).
+
+The RAD Linker is primarily optimized to handle huge linking projects - in our test cases (where debug info is multiple gigabytes), we see 50% faster link times. 
+
+The command line syntax is fully compatible with MSVC and you can get a full list of implemented switches from `/help`.
+
+Our current designed-for use case for the linker is to help with the compile-debug cycle of huge projects. We don't yet have support for dead-code-elimination or link-time-optimizations, but these features are on the road map.
+
+By default, the RAD linker spawns as many threads as there are cores, so if you plan to run multiple linkers in parallel, you can limit the number of thread workers via `/rad_workers`.
+
+We also have support for large memory pages, which, when enabled, reduce link time by
+another 25%. To link with large pages, you need to explicitly request them via `/rad_large_pages`. Large pages are off by default, since Windows support for large pages is a bit buggy - we recommend they only be used in Docker or VM images where the environment is reset after each link. In a standard Windows environment, using large pages otherwise will fragment memory quickly forcing a reboot. We are working on a Linux port of the linker that will be able to build with large pages robustly.
+
+## Short Term Roadmap
+- Porting linker to Linux (for Windows executables, just running on Linux).
+- Debug info features
+  - Get DWARF debug info converter up-and-running.
+  - Smooth out rough edges in RADDBGI builder.
+  - Improve build speed further (especially for tiny and mid sizes projects).
+- Other features to come
+  - Dead-code-elimination via `/opt:ref`.
+  - Link Time Optimizations with the help of clang (we won't support LTCG from MSVC compiler since it is undocumented).
+
+## To build the RAD Linker
+- Setup development environment, [see](#Development-Setup-Instructions)
+- Run `build linker release` or if you have clang installed `build linker release clang`. We favor latter option for better code generation.
+
+If build was successful linker executable will be placed in `build` folder under `radlink.exe`.
+
+## Benchmarks
+
+![AMD Ryzen Threadripper PRO 3995WX 64-Cores, 256 GiB RAM (Windows x64)](https://github.com/user-attachments/assets/39d95fdf-9f0b-45d3-9fb8-90d8ac624168)
+
+---
+
 ## Top-Level Directory Descriptions
 
 - `data`: Small binary files which are used when building, either to embed

build.bat

@@ -104,6 +104,7 @@ if "%rdi_from_pdb%"=="1"               set didbuild=1 && %compile% ..\src\rdi_fr
 if "%rdi_from_dwarf%"=="1"             set didbuild=1 && %compile% ..\src\rdi_from_dwarf\rdi_from_dwarf.c                    %compile_link% %out%rdi_from_dwarf.exe || exit /b 1
 if "%rdi_dump%"=="1"                   set didbuild=1 && %compile% ..\src\rdi_dump\rdi_dump_main.c                           %compile_link% %out%rdi_dump.exe || exit /b 1
 if "%rdi_breakpad_from_pdb%"=="1"      set didbuild=1 && %compile% ..\src\rdi_breakpad_from_pdb\rdi_breakpad_from_pdb_main.c %compile_link% %out%rdi_breakpad_from_pdb.exe || exit /b 1
+if "%radlink%"=="1"                    set didbuild=1 && %compile%  ..\src\linker\lnk.c                                      %compile_link% %out%radlink.exe || exit /b 1
 if "%tester%"=="1"                     set didbuild=1 && %compile% ..\src\tester\tester_main.c                               %compile_link% %out%tester.exe || exit /b 1
 if "%ryan_scratch%"=="1"               set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c                             %compile_link% %out%ryan_scratch.exe || exit /b 1
 if "%mule_main%"=="1"                  set didbuild=1 && del vc*.pdb mule*.pdb && %compile_release% %only_compile% ..\src\mule\mule_inline.cpp && %compile_release% %only_compile% ..\src\mule\mule_o2.cpp && %compile_debug% %EHsc% ..\src\mule\mule_main.cpp ..\src\mule\mule_c.c mule_inline.obj mule_o2.obj %compile_link% %no_aslr% %out%mule_main.exe || exit /b 1

src/base/base_arena.c

@@ -58,6 +58,10 @@ arena_alloc_(ArenaParams *params)
   arena->pos = ARENA_HEADER_SIZE;
   arena->cmt = commit_size;
   arena->res = reserve_size;
+#if ARENA_FREE_LIST
+  arena->free_size = 0;
+  arena->free_last = 0;
+#endif
   AsanPoisonMemoryRegion(base, commit_size);
   AsanUnpoisonMemoryRegion(base, ARENA_HEADER_SIZE);
   return arena;
@@ -85,18 +89,45 @@ arena_push(Arena *arena, U64 size, U64 align)
   // rjf: chain, if needed
   if(current->res < pos_pst && !(arena->flags & ArenaFlag_NoChain))
   {
-    U64 res_size = current->res_size;
-    U64 cmt_size = current->cmt_size;
-    if(size + ARENA_HEADER_SIZE > res_size)
+    Arena *new_block = 0;
+
+#if ARENA_FREE_LIST
+    Arena *prev_block;
+    for(new_block = arena->free_last, prev_block = 0; new_block != 0; prev_block = new_block, new_block = new_block->prev)
     {
-      res_size = size + ARENA_HEADER_SIZE;
-      cmt_size = size + ARENA_HEADER_SIZE;
+      if(new_block->res >= AlignPow2(size, align))
+      {
+        if(prev_block)
+        {
+          prev_block->prev = new_block->prev;
+        }
+        else
+        {
+          arena->free_last = new_block->prev;
+        }
+        arena->free_size -= new_block->res_size;
+		AsanUnpoisonMemoryRegion((U8*)new_block + ARENA_HEADER_SIZE, new_block->res_size - ARENA_HEADER_SIZE);
+        break;
+      }
     }
-    Arena *new_block = arena_alloc(.reserve_size = res_size,
-                                   .commit_size = cmt_size,
-                                   .flags = current->flags);
-    new_block->base_pos = current->base_pos + current->res;
-    SLLStackPush_N(arena->current, new_block, prev);
+#endif
+
+    if(new_block == 0)
+    {
+      U64 res_size = current->res_size;
+      U64 cmt_size = current->cmt_size;
+      if(size + ARENA_HEADER_SIZE > res_size)
+      {
+        res_size = size + ARENA_HEADER_SIZE;
+        cmt_size = size + ARENA_HEADER_SIZE;
+      }
+      new_block = arena_alloc(.reserve_size = res_size,
+                              .commit_size = cmt_size,
+                              .flags = current->flags);
+      new_block->base_pos = current->base_pos + current->res;
+      SLLStackPush_N(arena->current, new_block, prev);
+    }
+
     current = new_block;
     pos_pre = AlignPow2(current->pos, align);
     pos_pst = pos_pre + size;
@@ -155,11 +186,23 @@ arena_pop_to(Arena *arena, U64 pos)
 {
   U64 big_pos = ClampBot(ARENA_HEADER_SIZE, pos);
   Arena *current = arena->current;
+
+#if ARENA_FREE_LIST
+  for(Arena *prev = 0; current->base_pos >= big_pos; current = prev)
+  {
+    prev = current->prev;
+    current->pos = ARENA_HEADER_SIZE;
+    arena->free_size += current->res_size;
+    SLLStackPush_N(arena->free_last, current, prev);
+    AsanPoisonMemoryRegion((U8*)current + ARENA_HEADER_SIZE, current->res_size - ARENA_HEADER_SIZE);
+  }
+#else
   for(Arena *prev = 0; current->base_pos >= big_pos; current = prev)
   {
     prev = current->prev;
     os_release(current, current->res);
   }
+#endif
   arena->current = current;
   U64 new_pos = big_pos - current->base_pos;
   AssertAlways(new_pos <= current->pos);

src/base/base_arena.h

@@ -7,7 +7,11 @@
 ////////////////////////////////
 //~ rjf: Constants
 
-#define ARENA_HEADER_SIZE 64
+#if ARENA_FREE_LIST
+# define ARENA_HEADER_SIZE 128
+#else
+# define ARENA_HEADER_SIZE 64
+#endif
 
 ////////////////////////////////
 //~ rjf: Types
@@ -40,6 +44,10 @@ struct Arena
   U64 pos;
   U64 cmt;
   U64 res;
+#if ARENA_FREE_LIST
+  U64 free_size;
+  Arena *free_last;
+#endif
 };
 StaticAssert(sizeof(Arena) <= ARENA_HEADER_SIZE, arena_header_size_check);
 

src/base/base_strings.h

@@ -86,6 +86,7 @@ enum
 
 typedef enum PathStyle
 {
+  PathStyle_Null,
   PathStyle_Relative,
   PathStyle_WindowsAbsolute,
   PathStyle_UnixAbsolute,

src/codeview/codeview.c

@@ -80,6 +80,13 @@ cv_numeric_from_data_range(U8 *first, U8 *opl)
   return result;
 }
 
+internal U64
+cv_read_numeric(String8 data, U64 offset, CV_NumericParsed *out)
+{
+  *out = cv_numeric_from_data_range(data.str + offset, data.str + data.size);
+  return out->encoded_size;
+}
+
 internal B32
 cv_numeric_fits_in_u64(CV_NumericParsed *num)
 {
@@ -411,7 +418,7 @@ cv_leaf_from_data(Arena *arena, String8 leaf_data, CV_TypeId itype_first)
 //~ CodeView C13 Parser Functions
 
 internal CV_C13Parsed *
-cv_c13_parsed_from_data(Arena *arena, String8 c13_data, PDB_Strtbl *strtbl, PDB_CoffSectionArray *sections)
+cv_c13_parsed_from_data(Arena *arena, String8 c13_data, String8 strtbl, COFF_SectionHeaderArray sections)
 {
   ProfBeginFunction();
 
@@ -488,10 +495,10 @@ cv_c13_parsed_from_data(Arena *arena, String8 c13_data, PDB_Strtbl *strtbl, PDB_
         B32 has_cols = !!(hdr->flags & CV_C13SubSecLinesFlag_HasColumns);
         U64 secrel_off = hdr->sec_off;
         U64 secrel_opl = secrel_off + hdr->len;
-        U64 sec_base_off = sections->sections[sec_idx - 1].voff;
+        U64 sec_base_off = sections.v[sec_idx - 1].voff;
 
         // rjf: bad section index -> skip
-        if(sec_idx < 1 || sections->count < sec_idx)
+        if(sec_idx < 1 || sections.count < sec_idx)
         {
           continue;
         }
@@ -511,7 +518,8 @@ cv_c13_parsed_from_data(Arena *arena, String8 c13_data, PDB_Strtbl *strtbl, PDB_
           {
             CV_C13Checksum *checksum = (CV_C13Checksum*)(c13_data.str + file_chksms->off + file_off);
             U32 name_off = checksum->name_off;
-            file_name = pdb_strtbl_string_from_off(strtbl, name_off);
+            file_name =  str8_cstring_capped((char*)(strtbl.str + name_off),
+                                            (char*)(strtbl.str + strtbl.size));
           }
 
           // array layouts
@@ -587,7 +595,8 @@ cv_c13_parsed_from_data(Arena *arena, String8 c13_data, PDB_Strtbl *strtbl, PDB_
           {
             CV_C13Checksum *checksum = (CV_C13Checksum*)(c13_data.str + file_chksms->off + hdr->file_off);
             U32 name_off = checksum->name_off;
-            file_name = pdb_strtbl_string_from_off(strtbl, name_off);
+            file_name =  str8_cstring_capped((char*)(strtbl.str + name_off),
+                                            (char*)(strtbl.str + strtbl.size));
           }
 
           // rjf: parse extra files
@@ -608,6 +617,7 @@ cv_c13_parsed_from_data(Arena *arena, String8 c13_data, PDB_Strtbl *strtbl, PDB_
           SLLQueuePush(node->inlinee_lines_first, node->inlinee_lines_last, n);
           n->v.inlinee          = hdr->inlinee;
           n->v.file_name        = file_name;
+		  n->v.file_off         = hdr->file_off;
           n->v.first_source_ln  = hdr->first_source_ln;
           n->v.extra_file_count = extra_file_count;
           n->v.extra_files      = extra_files;