processless debug info loading/unloading; separate modules from dbg infos in eval; keep dbg infos around after debugging via config, correllate to new modules, evict when necessary, when generating new versions

Author: ryanfleury

src/ctrl/ctrl_core.c

@@ -3189,7 +3189,7 @@ ctrl_thread__entry_point(void *p)
             CTRL_Entity *module = ctrl_entity_from_handle(entity_ctx, msg->entity);
             CTRL_Entity *debug_info_path = ctrl_entity_child_from_kind(module, CTRL_EntityKind_DebugInfoPath);
             DI_Key old_dbgi_key = di_key_from_path_timestamp(debug_info_path->string, debug_info_path->timestamp);
-            di_close(old_dbgi_key);
+            di_close(old_dbgi_key, 0);
             MutexScopeW(ctrl_state->ctrl_thread_entity_ctx_rw_mutex)
             {
               ctrl_entity_equip_string(ctrl_state->ctrl_thread_entity_store, debug_info_path, path_normalized_from_string(scratch.arena, path));
@@ -4112,7 +4112,7 @@ ctrl_thread__next_dmn_event(Arena *arena, DMN_CtrlCtx *ctrl_ctx, CTRL_Msg *msg,
       out_evt->entity     = module_handle;
       out_evt->string     = module_path;
       DI_Key dbgi_key = ctrl_dbgi_key_from_module(module_ent);
-      di_close(dbgi_key);
+      di_close(dbgi_key, 0);
     }break;
     case DMN_EventKind_DebugString:
     {
@@ -4274,15 +4274,19 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
   U64 thread_rip_voff = ctrl_voff_from_vaddr(module, thread_rip_vaddr);
 
   //////////////////////////////
-  //- rjf: gather evaluation modules
+  //- rjf: gather evaluation debug infos & modules
   //
   U64 eval_modules_count = Max(1, entity_ctx->entity_kind_counts[CTRL_EntityKind_Module]);
   E_Module *eval_modules = push_array(arena, E_Module, eval_modules_count);
   E_Module *eval_modules_primary = &eval_modules[0];
-  eval_modules_primary->rdi = &rdi_parsed_nil;
   eval_modules_primary->vaddr_range = r1u64(0, max_U64);
+  U64 eval_dbg_infos_count = Max(1, entity_ctx->entity_kind_counts[CTRL_EntityKind_Module]);
+  E_DbgInfo *eval_dbg_infos = push_array(arena, E_DbgInfo, eval_dbg_infos_count);
+  E_DbgInfo *eval_dbg_infos_primary = &eval_dbg_infos[0];
+  MemoryCopyStruct(eval_dbg_infos_primary, &e_dbg_info_nil);
   {
     U64 eval_module_idx = 0;
+    U64 eval_dbg_info_idx = 0;
     for(CTRL_Entity *machine = entity_ctx->root->first;
         machine != &ctrl_entity_nil;
         machine = machine->next)
@@ -4392,10 +4396,18 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
             rdi = di_rdi_from_key(scope->access, dbgi_key, 1, max_U64);
           }
 
+          //- rjf: fill debug info
+          eval_dbg_infos[eval_dbg_info_idx].dbgi_key = dbgi_key;
+          eval_dbg_infos[eval_dbg_info_idx].rdi      = rdi;
+          if(mod == module)
+          {
+            eval_dbg_infos_primary = &eval_dbg_infos[eval_dbg_info_idx];
+          }
+          eval_dbg_info_idx += 1;
+          
           //- rjf: fill evaluation module info
           eval_modules[eval_module_idx].arch        = arch;
-          eval_modules[eval_module_idx].dbgi_key    = dbgi_key;
-          eval_modules[eval_module_idx].rdi         = rdi;
+          eval_modules[eval_module_idx].dbg_info_num= (U32)eval_dbg_info_idx;
           eval_modules[eval_module_idx].vaddr_range = mod->vaddr_range;
           eval_modules[eval_module_idx].space       = e_space_make(CTRL_EvalSpaceKind_Entity);
           eval_modules[eval_module_idx].space.u64_0 = (U64)process;
@@ -4427,6 +4439,11 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
     ctx->thread_reg_space = e_space_make(CTRL_EvalSpaceKind_Entity);
     ctx->thread_reg_space.u64_0 = (U64)thread;
 
+    //- rjf: fill debug infos
+    ctx->dbg_infos        = eval_dbg_infos;
+    ctx->dbg_infos_count  = eval_dbg_infos_count;
+    ctx->primary_dbg_info = eval_dbg_infos_primary;
+    
     //- rjf: fill modules
     ctx->modules        = eval_modules;
     ctx->modules_count  = eval_modules_count;
@@ -4445,8 +4462,8 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
     E_IRCtx *ctx = &scope->ir_ctx;
     ctx->regs_map      = ctrl_string2reg_from_arch(arch);
     ctx->reg_alias_map = ctrl_string2alias_from_arch(arch);
-    ctx->locals_map    = e_push_locals_map_from_rdi_voff(arena, eval_modules_primary->rdi, thread_rip_voff);
-    ctx->member_map    = e_push_member_map_from_rdi_voff(arena, eval_modules_primary->rdi, thread_rip_voff);
+    ctx->locals_map    = e_push_locals_map_from_rdi_voff(arena, eval_dbg_infos_primary->rdi, thread_rip_voff);
+    ctx->member_map    = e_push_member_map_from_rdi_voff(arena, eval_dbg_infos_primary->rdi, thread_rip_voff);
     ctx->macro_map     = push_array(arena, E_String2ExprMap, 1);
     ctx->macro_map[0]  = e_string2expr_map_make(arena, 512);
     ctx->auto_hook_map = push_array(arena, E_AutoHookMap, 1);
@@ -4469,7 +4486,7 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
     // TODO(rjf): need to compute this out here somehow... ctx->frame_base[0] = ;
     ctx->tls_base      = push_array(arena, U64, 1);
   }
-  e_select_interpret_ctx(&scope->interpret_ctx, eval_modules_primary->rdi, thread_rip_voff);
+  e_select_interpret_ctx(&scope->interpret_ctx, eval_dbg_infos_primary->rdi, thread_rip_voff);
 
   ProfEnd();
   return scope;

src/dbg_engine/dbg_engine_core.c

@@ -1500,6 +1500,16 @@ d_tick(Arena *arena, D_TargetArray *targets, D_BreakpointArray *breakpoints, D_P
           evt->code = event->u64_code;
         }break;
 
+        case CTRL_EventKind_NewModule:
+        {
+          D_EventNode *n = push_array(arena, D_EventNode, 1);
+          SLLQueuePush(result.first, result.last, n);
+          result.count += 1;
+          D_Event *evt = &n->v;
+          evt->kind = D_EventKind_ModuleLoad;
+          evt->module = event->entity;
+        }break;
+        
         //- rjf: debug strings
 
         case CTRL_EventKind_DebugString:

src/dbg_engine/dbg_engine_core.h

@@ -86,6 +86,7 @@ struct D_TrapNet
 typedef enum D_EventKind
 {
   D_EventKind_Null,
+  D_EventKind_ModuleLoad,
   D_EventKind_ProcessEnd,
   D_EventKind_Stop,
   D_EventKind_COUNT
@@ -107,6 +108,7 @@ struct D_Event
 {
   D_EventKind kind;
   D_EventCause cause;
+  CTRL_Handle module;
   CTRL_Handle thread;
   U64 vaddr;
   U64 code;

src/dbg_info/dbg_info.c

@@ -313,8 +313,8 @@ di_open(DI_Key key)
 }
 
 internal void
-di_close(DI_Key key)
-{
+di_close(DI_Key key, B32 force_closed)
+{
   //- rjf: unpack key
   U64 hash = u64_hash_from_str8(str8_struct(&key));
   U64 slot_idx = hash%di_shared->slots_count;
@@ -340,8 +340,15 @@ di_close(DI_Key key)
       }
     }
     if(node)
-    {
-      node->refcount -= 1;
+    {
+      if(force_closed)
+      {
+        node->refcount = 0;
+      }
+      else
+      {
+        node->refcount -= 1;
+      }
       if(node->refcount == 0)
       {
         for(;;)
@@ -377,7 +384,7 @@ di_close(DI_Key key)
     {
       arena_release(arena);
     }
-  }
+  }
 }
 
 ////////////////////////////////
@@ -754,8 +761,31 @@ di_async_tick(void)
           threads_available = (max_threads >= needed_threads);
         }
 
+        //- rjf: if this conversion will overwrite an RDI we already have in cache,
+        // then we need to evict the old one from the cache.
+        B32 ready_to_launch_conversion = (threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
+        if(ready_to_launch_conversion)
+        {
+          U64 path2key_hash = u64_hash_from_str8(og_path);
+          U64 path2key_slot_idx = path2key_hash%di_shared->path2key_slots_count;
+          DI_KeySlot *path2key_slot = &di_shared->path2key_slots[path2key_slot_idx];
+          Stripe *path2key_stripe = stripe_from_slot_idx(&di_shared->path2key_stripes, path2key_slot_idx);
+          RWMutexScope(path2key_stripe->rw_mutex, 0)
+          {
+            // NOTE(rjf): we need to iterate from last -> first, since we want to evict the
+            // most recent key.
+            for(DI_KeyPathNode *n = path2key_slot->last; n != 0; n = n->prev)
+            {
+              if(str8_match(n->path, og_path, 0) && !di_key_match(key, n->key))
+              {
+                di_close(n->key, 1);
+              }
+            }
+          }
+        }
+        
         //- rjf: launch conversion processes
-        if(threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active)
+        if(ready_to_launch_conversion)
         {
           B32 should_compress = 0;
           OS_ProcessLaunchParams params = {0};
@@ -960,8 +990,11 @@ di_async_tick(void)
             node->arena = rdi_parsed_arena;
             MemoryCopyStruct(&node->rdi, &rdi_parsed);
             node->completion_count += 1;
-            node->working_count -= 1;
-            ins_atomic_u64_inc_eval(&di_shared->load_gen);
+            node->working_count -= 1;
+            if(node->rdi.raw_data_size != 0)
+            {
+              ins_atomic_u64_inc_eval(&di_shared->load_gen);
+            }
             ins_atomic_u64_inc_eval(&di_shared->load_count);
           }
           else

src/dbg_info/dbg_info.h

@@ -323,7 +323,7 @@ internal DI_Key di_key_from_path_timestamp(String8 path, U64 min_timestamp);
 //~ rjf: Debug Info Opening / Closing
 
 internal void di_open(DI_Key key);
-internal void di_close(DI_Key key);
+internal void di_close(DI_Key key, B32 force_closed);
 
 ////////////////////////////////
 //~ rjf: Debug Info Lookups

src/eval/eval_core.c

@@ -676,8 +676,10 @@ internal void
 e_select_base_ctx(E_BaseCtx *ctx)
 {
   //- rjf: select base context
-  if(ctx->modules == 0)        { ctx->modules = &e_module_nil; }
-  if(ctx->primary_module == 0) { ctx->primary_module = &e_module_nil; }
+  if(ctx->modules == 0)          { ctx->modules = &e_module_nil; }
+  if(ctx->primary_module == 0)   { ctx->primary_module = &e_module_nil; }
+  if(ctx->dbg_infos == 0)        { ctx->dbg_infos = &e_dbg_info_nil; }
+  if(ctx->primary_dbg_info == 0) { ctx->primary_dbg_info = &e_dbg_info_nil; }
   e_base_ctx = ctx;
 
   //- rjf: reset the evaluation cache
@@ -718,7 +720,7 @@ e_select_base_ctx(E_BaseCtx *ctx)
                                                .id_from_num = E_TYPE_EXPAND_ID_FROM_NUM_FUNCTION_NAME(folder),
                                                .num_from_id = E_TYPE_EXPAND_NUM_FROM_ID_FUNCTION_NAME(folder),
                                              });
-  e_cache->thread_ip_procedure = rdi_procedure_from_voff(e_base_ctx->primary_module->rdi, e_base_ctx->thread_ip_voff);
+  e_cache->thread_ip_procedure = rdi_procedure_from_voff(e_base_ctx->primary_dbg_info->rdi, e_base_ctx->thread_ip_voff);
   e_cache->used_expr_map = push_array(e_cache->arena, E_UsedExprMap, 1);
   e_cache->used_expr_map->slots_count = 64;
   e_cache->used_expr_map->slots = push_array(e_cache->arena, E_UsedExprSlot, e_cache->used_expr_map->slots_count);
@@ -744,6 +746,20 @@ e_select_ir_ctx(E_IRCtx *ctx)
   e_ir_ctx = ctx;
 }
 
+////////////////////////////////
+//~ rjf: Context Accessors
+
+internal E_DbgInfo *
+e_dbg_info_from_module(E_Module *module)
+{
+  E_DbgInfo *result = &e_dbg_info_nil;
+  if(0 < module->dbg_info_num && module->dbg_info_num <= e_base_ctx->dbg_infos_count)
+  {
+    result = &e_base_ctx->dbg_infos[module->dbg_info_num-1];
+  }
+  return result;
+}
+
 ////////////////////////////////
 //~ rjf: Cache Accessing Functions
 

src/eval/eval_core.h

@@ -564,15 +564,24 @@ struct E_ConsTypeSlot
   E_ConsTypeNode *last;
 };
 
+////////////////////////////////
+//~ rjf: Debug Info
+
+typedef struct E_DbgInfo E_DbgInfo;
+struct E_DbgInfo
+{
+  DI_Key dbgi_key;
+  RDI_Parsed *rdi;
+};
+
 ////////////////////////////////
 //~ rjf: Modules
 
 typedef struct E_Module E_Module;
 struct E_Module
 {
-  DI_Key dbgi_key;
-  RDI_Parsed *rdi;
   Rng1U64 vaddr_range;
+  U32 dbg_info_num;
   Arch arch;
   E_Space space;
 };
@@ -760,6 +769,11 @@ struct E_BaseCtx
   Arch thread_arch;
   U64 thread_unwind_count;
 
+  // rjf: debug infos
+  E_DbgInfo *dbg_infos;
+  U64 dbg_infos_count;
+  E_DbgInfo *primary_dbg_info;
+  
   // rjf: modules
   E_Module *modules;
   U64 modules_count;
@@ -1111,7 +1125,8 @@ read_only global E_String2ExprMap e_string2expr_map_nil = {0};
 read_only global E_Expr e_expr_nil = {&e_expr_nil, &e_expr_nil, &e_expr_nil, &e_expr_nil, &e_expr_nil};
 read_only global E_IRNode e_irnode_nil = {&e_irnode_nil, &e_irnode_nil, &e_irnode_nil};
 read_only global E_Eval e_eval_nil = {{0}, {0}, {0}, &e_expr_nil, {&e_irnode_nil}};
-read_only global E_Module e_module_nil = {{0}, &rdi_parsed_nil};
+read_only global E_DbgInfo e_dbg_info_nil = {{0}, &rdi_parsed_nil};
+read_only global E_Module e_module_nil = {0};
 read_only global E_CacheBundle e_cache_bundle_nil = {0, {0}, {0}, {0}, {{0}, 0, &e_expr_nil, &e_expr_nil}, {&e_irnode_nil}};
 thread_static E_BaseCtx *e_base_ctx = 0;
 thread_static E_IRCtx *e_ir_ctx = 0;
@@ -1203,6 +1218,11 @@ internal void e_select_cache(E_Cache *cache);
 internal void e_select_base_ctx(E_BaseCtx *ctx);
 internal void e_select_ir_ctx(E_IRCtx *ctx);
 
+////////////////////////////////
+//~ rjf: Context Accessors
+
+internal E_DbgInfo *e_dbg_info_from_module(E_Module *module);
+
 ////////////////////////////////
 //~ rjf: Base Cache Accessing Functions
 //