Skip to content

Commit 66cf26e

Browse files
keevankeevan
authored
Merge pull request #855 from catalyst/additional-goodies-csv-sftp-perf-sql-and-more
additional goodies csv sftp perf sql and more
2 parents b8819bf + f7485ef commit 66cf26e

16 files changed

+460
-162
lines changed

‎classes/form/step_form.php

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,11 +107,22 @@ public function definition() {
107107
$select->setMultiple(true);
108108

109109
$dataflow = new dataflow($dataflowid);
110-
$varroot = $dataflow->get_variables_root();
111110

112-
// List all the available fields available for configuration, in dot syntax.
113-
$mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'),
114-
$this->prepare_available_fields($varroot->get()));
111+
try {
112+
$varroot = $dataflow->get_variables_root();
113+
114+
// List all the available fields available for configuration, in dot syntax.
115+
$mform->addElement(
116+
'static',
117+
'fields',
118+
get_string('available_fields', 'tool_dataflows'),
119+
$this->prepare_available_fields($varroot->get())
120+
);
121+
} catch (\Throwable $e) {
122+
global $OUTPUT;
123+
$errtext = $OUTPUT->notification($e->getMessage());
124+
$mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext);
125+
}
115126

116127
// Check and set custom form inputs if required. Defaulting to a
117128
// textarea config input for those not yet configured.

‎classes/local/execution/engine.php

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat
211211

212212
// Find the flow blocks.
213213
$this->create_flow_caps();
214+
215+
// Make the runid available to the flow.
216+
if (!$this->isdryrun) {
217+
$variables = $this->get_variables();
218+
$variables->set('run.name', $this->run->name);
219+
$variables->set('run.id', $this->run->id);
220+
}
214221
}
215222

216223
/**
@@ -792,7 +799,7 @@ private function setup_logging() {
792799

793800
// Dataflow run logger.
794801
// Type: FILE_PER_RUN
795-
// e.g. '[dataroot]/tool_dataflows/3/20060102150405-21.log' as the path.
802+
// e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path.
796803
if (isset($loghandlers[log_handler::FILE_PER_RUN])) {
797804
$dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
798805
'tool_dataflows' . DIRECTORY_SEPARATOR .

‎classes/local/step/connector_set_multiple_variables.php

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
// This file is part of Moodle - https://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace tool_dataflows\local\step;
18+
19+
/**
20+
* Set multiple variables connector step
21+
*
22+
* @package tool_dataflows
23+
* @author Kevin Pham <[email protected]>
24+
* @copyright Catalyst IT, 2023
25+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26+
*/
27+
class connector_set_multiple_variables extends connector_step {
28+
use set_multiple_variables_trait;
29+
30+
/** @var int[] number of output flows (min, max). */
31+
protected $outputflows = [0, 1];
32+
33+
/** @var int[] number of output connectors (min, max). */
34+
protected $outputconnectors = [0, 1];
35+
}

‎classes/local/step/connector_sql.php

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?php
2+
// This file is part of Moodle - https://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace tool_dataflows\local\step;
18+
19+
/**
20+
* SQL connector step
21+
*
22+
* @package tool_dataflows
23+
* @author Kevin Pham <[email protected]>
24+
* @copyright Catalyst IT, 2023
25+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26+
*/
27+
class connector_sql extends connector_step {
28+
use sql_trait;
29+
}

‎classes/local/step/copy_file_trait.php

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,9 @@ public function execute($input = null) {
7272
$todirectory = dirname($to);
7373
if (!file_exists($todirectory)) {
7474
$this->log("Creating a directory at {$todirectory}");
75-
mkdir($todirectory, $CFG->directorypermissions, true);
75+
if (!mkdir($todirectory, $CFG->directorypermissions, true)) {
76+
throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory);
77+
}
7678
}
7779

7880
// Attempt to copy the file to the destination.
@@ -112,10 +114,12 @@ public function execute($input = null) {
112114
private function copy(string $from, string $to) {
113115
$this->log("Copying $from to $to");
114116
if (!copy($from, $to)) {
115-
throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [
116-
'from' => $from,
117-
'to' => $to,
118-
]);
117+
throw new \moodle_exception(
118+
'flow_copy_file:copy_failed',
119+
'tool_dataflows',
120+
'',
121+
(object) ['from' => $from, 'to' => $to]
122+
);
119123
}
120124
}
121125

‎classes/local/step/flow_email.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@
2828
*/
2929
class flow_email extends flow_step {
3030

31+
/** @var int[] number of output flows (min, max). */
32+
protected $outputflows = [0, 1];
33+
34+
/** @var int[] number of output connectors (min, max) */
35+
protected $outputconnectors = [0, 1];
36+
3137
/** @var bool whether or not this step type (potentially) contains a side effect or not */
3238
protected $hassideeffect = true;
3339

‎classes/local/step/flow_set_multiple_variables.php

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
// This file is part of Moodle - http://moodle.org/
3+
//
4+
// Moodle is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// Moodle is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
16+
17+
namespace tool_dataflows\local\step;
18+
19+
/**
20+
* Set multiple variables flow step
21+
*
22+
* @package tool_dataflows
23+
* @author Kevin Pham <[email protected]>
24+
* @copyright Catalyst IT, 2023
25+
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26+
*/
27+
class flow_set_multiple_variables extends flow_step {
28+
use set_multiple_variables_trait;
29+
30+
/** @var int[] number of output flows (min, max). */
31+
protected $outputflows = [0, 1];
32+
33+
/** @var int[] number of output connectors (min, max). */
34+
protected $outputconnectors = [0, 1];
35+
}

‎classes/local/step/flow_sql.php

Lines changed: 0 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -27,131 +27,4 @@
2727
class flow_sql extends flow_step {
2828
use sql_trait;
2929

30-
/**
31-
* Return the definition of the fields available in this form.
32-
*
33-
* @return array
34-
*/
35-
public static function form_define_fields(): array {
36-
return [
37-
'sql' => ['type' => PARAM_TEXT, 'required' => true],
38-
];
39-
}
40-
41-
/**
42-
* Allows each step type to determine a list of optional/required form
43-
* inputs for their configuration
44-
*
45-
* It's recommended you prefix the additional config related fields to avoid
46-
* conflicts with any existing fields.
47-
*
48-
* @param \MoodleQuickForm $mform
49-
*/
50-
public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
51-
// SQL example and inputs.
52-
$sqlexample = "
53-
SELECT id, username
54-
FROM {user}
55-
WHERE id > \${{steps.xyz.var.number}}
56-
ORDER BY id ASC
57-
LIMIT 10";
58-
$sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B"));
59-
$mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'),
60-
['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']);
61-
$mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples));
62-
}
63-
64-
/**
65-
* Allow steps to setup the form depending on current values.
66-
*
67-
* This method is called after definition(), data submission and set_data().
68-
* All form setup that is dependent on form values should go in here.
69-
*
70-
* @param \MoodleQuickForm $mform
71-
* @param \stdClass $data
72-
*/
73-
public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) {
74-
// Validate the data.
75-
$sqllinecount = count(explode(PHP_EOL, trim($data->config_sql)));
76-
77-
// Get the element.
78-
$element = $mform->getElement('config_sql');
79-
80-
// Update the element height based on min/max settings, but preserve
81-
// other existing rules.
82-
$attributes = $element->getAttributes();
83-
84-
// Set the rows at a minimum to the predefined amount in
85-
// form_add_custom_inputs, and expand as content grows up to a maximum.
86-
$attributes['rows'] = min(
87-
$attributes['max_rows'],
88-
max($attributes['rows'], $sqllinecount)
89-
);
90-
$element->setAttributes($attributes);
91-
}
92-
93-
/**
94-
* Execute configured query
95-
*
96-
* @param mixed $input
97-
* @return mixed
98-
* @throws \dml_read_exception when the SQL is not valid.
99-
*/
100-
public function execute($input = null) {
101-
global $DB;
102-
103-
// Construct the query.
104-
$variables = $this->get_variables();
105-
$config = $variables->get_raw('config');
106-
[$sql, $params] = $this->evaluate_expressions($config->sql);
107-
108-
// Now that we have the query, we want to get info on SQL keywords to figure out where to route the request.
109-
// This is not used for security, just to route the request via the correct pathway for readonly databases.
110-
$pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im';
111-
$matches = [];
112-
preg_match($pattern, $sql, $matches);
113-
114-
// Matches[0] contains the match. Fallthrough to default on no match.
115-
$token = $matches[0] ?? '';
116-
$emptydefault = new \stdClass();
117-
118-
switch(strtoupper($token)) {
119-
case 'SELECT':
120-
// Execute the query using get_records instead of get_record.
121-
// This is so we can expose the number of records returned which
122-
// can then be used by the dataflow in for e.g. a switch statement.
123-
$records = $DB->get_records_sql($sql, $params);
124-
125-
$variables->set('count', count($records));
126-
$invalidnum = ($records === false || count($records) !== 1);
127-
$data = $invalidnum ? $emptydefault : array_pop($records);
128-
$variables->set('data', $data);
129-
break;
130-
default:
131-
// Default to execute.
132-
$success = $DB->execute($sql, $params);
133-
134-
// We can't really do anything with the response except check for success.
135-
$variables->set('count', (int) $success);
136-
$variables->set('data', $emptydefault);
137-
break;
138-
}
139-
140-
return $input;
141-
}
142-
143-
/**
144-
* Validate the configuration settings.
145-
*
146-
* @param object $config
147-
* @return true|\lang_string[] true if valid, an array of errors otherwise
148-
*/
149-
public function validate_config($config) {
150-
$errors = [];
151-
if (empty($config->sql)) {
152-
$errors['config_sql'] = get_string('config_field_missing', 'tool_dataflows', 'sql', true);
153-
}
154-
155-
return empty($errors) ? true : $errors;
156-
}
15730
}

0 commit comments

Comments
 (0)