Add linear backoff in hook script

Use the _delay_ which comes from FinalizeResults to override the default Minion
delay behavior and force a linear backoff calculation based on the retries in
case of job failure and until it reaches the max retries. This should offer a
less agressive pattern, giving the Minion tasks more time to recover from
various issues and prevent overloading. Linear backoff is selected over
exponential as the later is already applied to the Minion tasks in general.

https://progress.opensuse.org/issues/181427

Co-authored-by: perlpunk  <[email protected]>

Author: d3flex

lib/OpenQA/Task/Job/HookScript.pm

@@ -13,16 +13,17 @@ sub _hook_script ($job, $hook, $openqa_job_id, $options) {
     my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job);
     my $timeout = $options->{timeout};
     my $kill_timeout = $options->{kill_timeout};
-    my $delay = $options->{delay};
     my $retries = $options->{retries};
+    my $delay = $options->{delay} * ($job->retries + 1);
     my $skip_rc = $options->{skip_rc};
 
     $ensure_task_retry_on_termination_signal_guard->abort(1);
     my ($rc, $out) = _run_hook($hook, $openqa_job_id, $timeout, $kill_timeout);
     $job->note(hook_cmd => $hook, hook_result => $out, hook_rc => $rc);
 
-    if ($retries && $skip_rc) {
-        $job->retry($delay ? {delay => $delay} : {}) if defined $rc && $rc == $skip_rc && $job->retries < $retries;
+    if ($retries && $skip_rc && defined $rc && $rc == $skip_rc && $job->retries < $retries) {
+        $job->app->log->debug("Job $job->id: Retrying $job->retries with linear delay: ${delay}s");
+        $job->retry($delay ? {delay => $delay} : {});
     }
 }
 

t/10-jobs.t

@@ -696,6 +696,39 @@ subtest 'carry over, including soft-fails' => sub {
             is $notes->{hook_rc}, 142, 'exit code of the hook cmd is as expected';
             is $job_info->{retries}, 1, 'hook script has been retried once because of delay';
         };
+
+        subtest 'There is a linear delay for each hook script which exits with 142' => sub {
+            $job->settings->create({key => '_TRIGGER_JOB_DONE_RETRIES', value => '4'});
+            $hooks->{job_done_hook_failed} = 'echo delayed && exit 142;';
+            $job->update({state => UPLOADING});
+            $job->discard_changes;
+            $job->done;
+            perform_minion_jobs($t->app->minion);
+            $job_info = $t->app->minion->jobs({tasks => ['hook_script']})->next;
+            is_deeply(
+                $job_info->{args}[2],
+                {delay => 60, retries => 4, skip_rc => 142, kill_timeout => '30s', timeout => '5m'},
+                'args contain expected parameters'
+            );
+            is $job_info->{retries}, 1, 'hook script has been already tried once since has been queued';
+            is $job_info->{state}, 'inactive', 'job ends up inactive due to retry';
+            note "we expect 4 retries * the default one minute delay";
+            my $linear_delay = $job_info->{delayed} - $job_info->{retried};
+            is $linear_delay, 60, 'default delay for first retry applied correctly';
+            # force some retries with delay 0 to avoid having to wait in the test
+            $t->app->minion->job($job_info->{id})->retry({delay => 0}) for 1, 2;
+            perform_minion_jobs($t->app->minion);
+            $job_info = $t->app->minion->jobs({tasks => ['hook_script']})->next;
+            is $job_info->{retries}, 4, 'hook script has been retried the expected number of times';
+            $linear_delay = $job_info->{delayed} - $job_info->{retried};
+            is $linear_delay, 240, 'linear delay calculated correctly for 4th retry';
+            note "Retry one last time to validate the final state";
+            $t->app->minion->job($job_info->{id})->retry({delay => 0});
+            perform_minion_jobs($t->app->minion);
+            $job_info = $t->app->minion->jobs({tasks => ['hook_script']})->next;
+            is $job_info->{retries}, 5, 'hook script retried for last time, exceeding the retry limit';
+            is $job_info->{state}, 'finished', 'hook script at the end of the retries has finished state';
+        };
     };
 };