Skip to content

Commit dbbffb4

Browse files
PeterburnettPeterburnett
committed
feat: Smart retries with delay + retry count for cron trigger
1 parent f467840 commit dbbffb4

File tree

10 files changed

+166
-5
lines changed

10 files changed

+166
-5
lines changed

‎classes/local/execution/engine.php

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,11 @@ public function abort(?\Throwable $reason = null) {
571571
if ($status !== self::STATUS_FINISHED && !in_array($status, self::STATUS_TERMINATORS)) {
572572
$this->set_current_step($enginestep);
573573
$enginestep->abort();
574+
} else {
575+
// We need to signal to finished steps that the dataflow is aborted. This may require handling seperate to the step abort.
576+
// This is done seperate to the finalise hook so that concerns are seperated for finalised vs aborted runs.
577+
$this->set_current_step($enginestep);
578+
$enginestep->dataflow_abort();
574579
}
575580
}
576581
foreach ($this->flowcaps as $enginestep) {

‎classes/local/execution/engine_step.php

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,14 @@ public function abort() {
144144
$this->steptype->on_abort();
145145
}
146146

147+
/**
148+
* Signal handler for a full dataflow abort.
149+
*/
150+
public function dataflow_abort() {
151+
$this->set_status(engine::STATUS_ABORTED);
152+
$this->steptype->on_dataflow_abort();
153+
}
154+
147155
/**
148156
* Attempt to execute the step.
149157
*

‎classes/local/scheduler.php

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public static function get_scheduled_times(int $stepid) {
5454
public static function set_scheduled_times(int $dataflowid, int $stepid, int $newtime, ?int $oldtime = null) {
5555
global $DB;
5656

57-
$obj = (object) ['nextruntime' => $newtime, 'dataflowid' => $dataflowid, 'stepid' => $stepid];
57+
$obj = (object) ['nextruntime' => $newtime, 'dataflowid' => $dataflowid, 'stepid' => $stepid, 'retrycount' => 0];
5858
if (!is_null($oldtime)) {
5959
$obj->lastruntime = $oldtime;
6060
}
@@ -67,6 +67,39 @@ public static function set_scheduled_times(int $dataflowid, int $stepid, int $ne
6767
}
6868
}
6969

70+
/**
71+
* Schedule a retry run. If the maximum retry count is reached, set to regular scheduled time and no retry count.
72+
*
73+
* @param int $dataflowid the flow id.
74+
* @param int $stepid the step trigger id.
75+
* @param int $retrytime when to run next on a retry.
76+
* @param int $scheduledtime when to run next if allowed retries are exhausted.
77+
* @param int $retriesallowed the amount of retries allowed before resuming regular schedule.
78+
*/
79+
public static function set_scheduled_retry(int $dataflowid, int $stepid, int $retrytime, int $scheduledtime, int $retriesallowed) {
80+
global $DB;
81+
82+
$schedule = $DB->get_record(self::TABLE, ['dataflowid' => $dataflowid, 'stepid' => $stepid]);
83+
84+
if (!$schedule) {
85+
// This method has been called incorrectly for a schedule that has never run or doesn't exist.
86+
// Just return early.
87+
return;
88+
}
89+
90+
if ($schedule->retrycount >= $retriesallowed) {
91+
// Allowed retries are exhausted. Set to regular schedule and no retries.
92+
$schedule->retrycount = 0;
93+
$schedule->nextruntime = $scheduledtime;
94+
} else {
95+
// Increment retry counter, and schedule the retry time.
96+
$schedule->retrycount = $schedule->retrycount + 1;
97+
$schedule->nextruntime = $retrytime;
98+
}
99+
100+
$DB->update_record(self::TABLE, $schedule);
101+
}
102+
70103
/**
71104
* Gets a list of dataflows and timestamps that are due to run based on the given reference time.
72105
*

‎classes/local/step/base_step.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,12 @@ public function on_initialise() {
587587
public function on_abort() {
588588
}
589589

590+
/**
591+
* Hook function that gets called when a dataflow has been aborted, at conclusion.
592+
*/
593+
public function on_dataflow_abort() {
594+
}
595+
590596
/**
591597
* Hook function that gets called when an engine step has been finalised.
592598
*/

‎classes/local/step/trigger_cron.php

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ public static function form_define_fields(): array {
4242
'day' => ['type' => PARAM_TEXT],
4343
'month' => ['type' => PARAM_TEXT],
4444
'dayofweek' => ['type' => PARAM_TEXT],
45+
'retryinterval' => ['type' => PARAM_INT],
46+
'retrycount' => ['type' => PARAM_INT],
4547
'disabled' => ['type' => PARAM_TEXT],
4648
];
4749
}
@@ -60,6 +62,9 @@ public function form_get_default_data(\stdClass $data): \stdClass {
6062
$data->{"config_$field"} = '*';
6163
}
6264
}
65+
$data->config_retryinterval = $data->config_retryinterval ?? 0;
66+
$data->config_retrycount = $data->config_retrycount ?? 0;
67+
6368
return $data;
6469
}
6570

@@ -128,6 +133,13 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
128133

129134
$mform->addGroup($crontab, 'crontab', get_string('trigger_cron:crontab', 'tool_dataflows'), ' ', false);
130135
$mform->addElement('static', 'crontab_desc', '', get_string('trigger_cron:crontab_desc', 'tool_dataflows'));
136+
137+
// Retry configurations.
138+
$mform->addElement('duration', 'config_retryinterval', get_string('trigger_cron:retryinterval', 'tool_dataflows'));
139+
$mform->setType('retryinterval', PARAM_INT);
140+
$mform->addElement('text', 'config_retrycount', get_string('trigger_cron:retrycount', 'tool_dataflows'));
141+
$mform->setType('retrycount', PARAM_INT);
142+
$mform->setDefault('retrycount', 0);
131143
}
132144

133145
/**
@@ -143,6 +155,13 @@ public function validate_config($config) {
143155
return ['crontab' => get_string('trigger_cron:invalid', 'tool_dataflows', '', true)];
144156
}
145157
}
158+
if ($config->retryinterval < 0) {
159+
return ['config_retryinterval' => get_string('trigger_cron:positive_retryinterval', 'tool_dataflows', null, true)];
160+
}
161+
if ($config->retrycount < 0) {
162+
return ['config_retrycount' => get_string('trigger_cron:positive_retryinterval', 'tool_dataflows', null, true)];
163+
}
164+
146165
return true;
147166
}
148167

@@ -276,8 +295,18 @@ public function on_finalise() {
276295
*/
277296
public function on_abort() {
278297
if (!$this->stepdef->dataflow->is_concurrency_enabled()) {
279-
// Reschedule on aborts.
280-
$this->reschedule();
298+
// Reschedule a retry on aborts.
299+
$this->reschedule_retry();
300+
}
301+
}
302+
303+
/**
304+
* Hook function that gets called when an engine step has been aborted.
305+
*/
306+
public function on_dataflow_abort() {
307+
if (!$this->stepdef->dataflow->is_concurrency_enabled()) {
308+
// Reschedule a retry on aborts.
309+
$this->reschedule_retry();
281310
}
282311
}
283312

@@ -295,6 +324,26 @@ protected function reschedule() {
295324
$newtime,
296325
$config->nextruntime ?? null
297326
);
327+
$this->log("Rescheduling dataflow to configured schedule.");
298328
}
299329
}
330+
331+
/**
332+
* Schedule a retry for this flow. If the maximum retries are reached, the regular schedule will be used.
333+
*/
334+
public function reschedule_retry() {
335+
$config = $this->get_variables()->get('config');
336+
$scheduledtime = $this->get_next_scheduled_time($config);
337+
$retrytime = time() + $config->retryinterval;
338+
339+
scheduler::set_scheduled_retry(
340+
$this->stepdef->dataflowid,
341+
$this->stepdef->id,
342+
$retrytime,
343+
$scheduledtime,
344+
$config->retrycount
345+
);
346+
347+
$this->log("Rescheduling dataflow to retry.");
348+
}
300349
}

‎db/install.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
<FIELD NAME="stepid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
6868
<FIELD NAME="lastruntime" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false" COMMENT="The time the dataflow was last scheduled to be run"/>
6969
<FIELD NAME="nextruntime" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="The time the dataflow is next scheduled to be run"/>
70+
<FIELD NAME="retrycount" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false" COMMENT="Count of attempted retries on the current dataflow. Reset on scheduling a fresh run."/>
7071
</FIELDS>
7172
<KEYS>
7273
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>

‎db/upgrade.php

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,21 @@ function xmldb_tool_dataflows_upgrade($oldversion) {
287287
upgrade_plugin_savepoint(true, 2023072100, 'tool', 'dataflows');
288288
}
289289

290+
if ($oldversion < 2024030201) {
291+
292+
// Define field retrycount to be added to tool_dataflows_schedule.
293+
$table = new xmldb_table('tool_dataflows_schedule');
294+
$field = new xmldb_field('retrycount', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, 0, 'nextruntime');
295+
296+
// Conditionally launch add field retrycount.
297+
if (!$dbman->field_exists($table, $field)) {
298+
$dbman->add_field($table, $field);
299+
}
300+
301+
// Dataflows savepoint reached.
302+
upgrade_plugin_savepoint(true, 2024030201, 'tool', 'dataflows');
303+
}
304+
290305
// Move log files that exist across to new format. Breaking change if any
291306
// dataflows implement logic based on these files based on filename format.
292307
if ($oldversion < 2023122201) {

‎lang/en/tool_dataflows.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,10 @@
396396
$string['trigger_cron:crontab_desc'] = 'The schedule is edited as five values: minute, hour, day, month and day of month, in that order. The values are in crontab format.';
397397
$string['trigger_cron:strftime_datetime'] = '%d %b %Y, %H:%M';
398398
$string['trigger_cron:next_run_time'] = 'Next run time: {$a}';
399+
$string['trigger_cron:retryinterval'] = 'Retry interval';
400+
$string['trigger_cron:retrycount'] = 'Number of retries';
401+
$string['trigger_cron:positive_retrycount'] = 'Number of retries must be positive or 0';
402+
$string['trigger_cron:positive_retryinterval'] = 'Retry interval must be positive or 0';
399403

400404
// Email notification.
401405
$string['connector_email:message'] = 'Message';

‎tests/tool_dataflows_scheduler_test.php

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
namespace tool_dataflows;
1818

19+
use Aws\Arn\Arn;
1920
use tool_dataflows\local\scheduler;
2021

2122
/**
@@ -70,6 +71,45 @@ public function test_update_next_scheduled_time() {
7071
$this->assertEquals((object) ['lastruntime' => 160, 'nextruntime' => 220], scheduler::get_scheduled_times(12));
7172
}
7273

74+
public function test_set_scheduled_retry() {
75+
global $DB;
76+
77+
// Retry cannot be called for a run that hasn't been regularly scheduled.
78+
scheduler::set_scheduled_retry(1, 1, 1, 1, 1);
79+
$this->assertFalse(scheduler::get_scheduled_times(1));
80+
81+
// Default 0.
82+
scheduler::set_scheduled_times(1, 1, 123, 1);
83+
$this->assertEquals(0, $DB->get_field(scheduler::TABLE, 'retrycount', ['stepid' => 1]));
84+
85+
// Schedule a retry when none are allowed.
86+
$regulartime = 555;
87+
$retrytime = 444;
88+
scheduler::set_scheduled_retry(1, 1, $retrytime, $regulartime, 0);
89+
$this->assertEquals((object) ['lastruntime' => 1, 'nextruntime' => $regulartime], scheduler::get_scheduled_times(1));
90+
91+
// Schedule a retry when retries are permitted.
92+
scheduler::set_scheduled_retry(1, 1, $retrytime, $regulartime, 2);
93+
$this->assertEquals((object) ['lastruntime' => 1, 'nextruntime' => $retrytime], scheduler::get_scheduled_times(1));
94+
95+
// Now run again and confirm counter has been incremented twice.
96+
scheduler::set_scheduled_retry(1, 1, $retrytime, $regulartime, 2);
97+
$this->assertEquals((object) ['lastruntime' => 1, 'nextruntime' => $retrytime], scheduler::get_scheduled_times(1));
98+
$this->assertEquals(2, $DB->get_field(scheduler::TABLE, 'retrycount', ['stepid' => 1]));
99+
100+
// Now attempt to schedule another retry. The counter should reset and go to regular time.
101+
scheduler::set_scheduled_retry(1, 1, $retrytime, $regulartime, 2);
102+
$this->assertEquals((object) ['lastruntime' => 1, 'nextruntime' => $regulartime], scheduler::get_scheduled_times(1));
103+
$this->assertEquals(0, $DB->get_field(scheduler::TABLE, 'retrycount', ['stepid' => 1]));
104+
105+
// Now confirm that if a successful run is registered while there are still retries left, the counter is reset.
106+
scheduler::set_scheduled_retry(1, 1, $retrytime, $regulartime, 2);
107+
$this->assertEquals((object) ['lastruntime' => 1, 'nextruntime' => $retrytime], scheduler::get_scheduled_times(1));
108+
$this->assertEquals(1, $DB->get_field(scheduler::TABLE, 'retrycount', ['stepid' => 1]));
109+
scheduler::set_scheduled_times(1, 1, $regulartime);
110+
$this->assertEquals(0, $DB->get_field(scheduler::TABLE, 'retrycount', ['stepid' => 1]));
111+
}
112+
73113
/**
74114
* Tests the get_due_dataflows() function.
75115
*

‎version.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525

2626
defined('MOODLE_INTERNAL') || die();
2727

28-
$plugin->version = 2023122201;
29-
$plugin->release = 2023122201;
28+
$plugin->version = 2024030201;
29+
$plugin->release = 2024030201;
3030
$plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0).
3131
$plugin->supported = [400, 402];
3232
// TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later.

0 commit comments

Comments
 (0)