Make test success criteria more strict than "let's observe a single async ID"

Author: szegedi

ts/test/test-time-profiler.ts

@@ -374,12 +374,12 @@ describe('Time Profiler', () => {
 
     const testStart = hrtime.bigint();
     const testDurationNanos = PROFILE_OPTIONS.durationMillis * 1_000_000;
+    const intervalNanos = PROFILE_OPTIONS.intervalMicros * 1_000;
     setTimeout(loop, 0);
 
     function loop() {
-      const loopDurationNanos = PROFILE_OPTIONS.intervalMicros * 1_000;
       const loopStart = hrtime.bigint();
-      while (hrtime.bigint() - loopStart < loopDurationNanos);
+      while (hrtime.bigint() - loopStart < intervalNanos);
       if (hrtime.bigint() - testStart < testDurationNanos) {
         setTimeout(loop, 0);
       } else {
@@ -389,14 +389,45 @@ describe('Time Profiler', () => {
 
     await done;
 
-    let asyncIdObserved = false;
+    const asyncIdObserved = new Set();
+    let sampleCount = 0;
+    let sampleCountWithAsyncID = 0;
     time.stop(false, ({context}: GenerateTimeLabelsArgs) => {
-      if (!asyncIdObserved && typeof context?.asyncId === 'number') {
-        asyncIdObserved = context?.asyncId !== -1;
+      if (typeof context?.asyncId === 'number') {
+        asyncIdObserved.add(context.asyncId);
+        sampleCountWithAsyncID++;
       }
+      sampleCount++;
       return {};
     });
-    assert(asyncIdObserved, 'Async ID was not observed');
+
+    // Below we set certain thresholds for test success. Note that in the extreme, the test could
+    // succeed even if we observed a single async ID with value other than -1. We are setting a
+    // somewhat higher bar, while still keeping it low enough to avoid flakiness.
+
+    const theoreticalSampleCount = testDurationNanos / intervalNanos;
+    // In local tests on my Mac I get around 90% of theoretical number. Still, we'll accept anything
+    // more than the quarter of the theoretical count so we don't flake out when tested on a busy
+    // machine.
+    const expectedSampleCount = theoreticalSampleCount / 4;
+    assert(
+      sampleCount >= expectedSampleCount,
+      `Observed ${sampleCount} samples, expected at least ${expectedSampleCount}`
+    );
+    // We'd expect at least 1/12 of samples to have async IDs. This is a very low number, but
+    // anything higher makes ASAN tests flaky.
+    const expectedSamplesWithAsyncID = sampleCount / 12;
+    assert(
+      sampleCountWithAsyncID >= expectedSamplesWithAsyncID,
+      `Observed ${sampleCountWithAsyncID} samples with async IDs, expected ${expectedSamplesWithAsyncID}`
+    );
+    // Similarly, we'd expect that the number of unique async IDs is at least 60% of all observed
+    // samples with async IDs.
+    const expectedOccurrences = (sampleCountWithAsyncID * 3) / 5;
+    assert(
+      asyncIdObserved.size >= expectedOccurrences,
+      `Observed ${asyncIdObserved.size} distinct async IDs, expected at least ${expectedOccurrences}`
+    );
   });
 
   describe('profile (w/ stubs)', () => {