Skip to content

Commit 297fd36

Browse files
committed
Detect external job interruption and log warning
1 parent 3486c54 commit 297fd36

File tree

1 file changed

+47
-0
lines changed

1 file changed

+47
-0
lines changed

src/Runner.Worker/JobRunner.cs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ public sealed class JobRunner : RunnerService, IJobRunner
3131
private IJobServerQueue _jobServerQueue;
3232
private RunnerSettings _runnerSettings;
3333
private ITempDirectoryManager _tempDirectoryManager;
34+
private const string InterruptedHookPath = "/opt/runs-on/hooks/interrupted";
35+
private readonly CancellationTokenSource _interruptedHookTokenSource = new();
36+
private Task _interruptedHookTask;
3437

3538
public async Task<TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
3639
{
@@ -216,6 +219,9 @@ public async Task<TaskResult> RunAsync(AgentJobRequestMessage message, Cancellat
216219
await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
217220
}
218221

222+
// Start monitoring for interrupted hook file
223+
_interruptedHookTask = MonitorInterruptedHookAsync(jobContext, _interruptedHookTokenSource.Token);
224+
219225
// Run all job steps
220226
Trace.Info("Run all job steps.");
221227
var stepsRunner = HostContext.GetService<IStepsRunner>();
@@ -256,6 +262,12 @@ public async Task<TaskResult> RunAsync(AgentJobRequestMessage message, Cancellat
256262
runnerShutdownRegistration = null;
257263
}
258264

265+
if (_interruptedHookTask != null)
266+
{
267+
_interruptedHookTokenSource.Cancel();
268+
await _interruptedHookTask;
269+
}
270+
259271
await ShutdownQueue(throwOnFailure: false);
260272
}
261273
}
@@ -546,5 +558,40 @@ private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipe
546558
Trace.Error($"Caught exception during runner version check: {ex}");
547559
}
548560
}
561+
562+
private async Task MonitorInterruptedHookAsync(IExecutionContext jobContext, CancellationToken token)
563+
{
564+
while (!token.IsCancellationRequested)
565+
{
566+
try
567+
{
568+
if (File.Exists(InterruptedHookPath))
569+
{
570+
// Add warning annotation
571+
var issue = new Issue
572+
{
573+
Type = IssueType.Warning,
574+
Message = "This job was interrupted by the runner infrastructure. Results may be incomplete."
575+
};
576+
jobContext.AddIssue(issue, new Dictionary<string, string>());
577+
578+
// Only warn once
579+
break;
580+
}
581+
582+
await Task.Delay(TimeSpan.FromSeconds(1), token);
583+
}
584+
catch (OperationCanceledException) when (token.IsCancellationRequested)
585+
{
586+
// Normal cancellation, ignore
587+
break;
588+
}
589+
catch (Exception ex)
590+
{
591+
// Log but continue monitoring
592+
Trace.Error($"Error checking interrupted hook file: {ex}");
593+
}
594+
}
595+
}
549596
}
550597
}

0 commit comments

Comments
 (0)