patch for delay race condition

Author: mogery

apps/api/src/controllers/v1/crawl-status.ts

@@ -22,7 +22,7 @@ import { configDotenv } from "dotenv";
 import type { Job, JobState, Queue } from "bullmq";
 import { logger } from "../../lib/logger";
 import { supabase_rr_service, supabase_service } from "../../services/supabase";
-import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
+import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";
 import { getJobFromGCS } from "../../lib/gcs-jobs";
 configDotenv();
 
@@ -163,6 +163,7 @@ export async function crawlStatusController(
     );
 
     const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));
+    const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));
 
     const validJobStatuses: [string, JobState | "unknown"][] = [];
     const validJobIDs: string[] = [];
@@ -171,6 +172,9 @@ export async function crawlStatusController(
       if (throttledJobsSet.has(id)) {
         validJobStatuses.push([id, "prioritized"]);
         validJobIDs.push(id);
+      } else if (status === "unknown" && activeJobsSet.has(id)) {
+        validJobStatuses.push([id, "active"]);
+        validJobIDs.push(id);
       } else if (
         status !== "failed" &&
         status !== "unknown"

apps/api/src/lib/concurrency-limit.ts

@@ -216,8 +216,7 @@ async function getNextConcurrentJob(teamId: string): Promise<{
   }
 
   for (const ignoredJob of ignoredJobs) {
-    const timeout = ignoredJob.timeout - Date.now();
-    await pushConcurrencyLimitedJob(teamId, ignoredJob.job, timeout);
+    await pushConcurrencyLimitedJob(teamId, ignoredJob.job, ignoredJob.timeout);
   }
 
   return finalJob;