feat: add an option to continue processing the csv file on known pars…

…e issues The only one is currently a count mismatch on header and the row fields
catalyst · Dec 21, 2023 · a996838 · a996838
1 parent f3df654
commit a996838
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 1 deletion.
diff --git a/classes/local/step/reader_csv.php b/classes/local/step/reader_csv.php
@@ -43,6 +43,7 @@ public static function form_define_fields(): array {
             'path' => ['type' => PARAM_TEXT, 'required' => true],
             'headers' => ['type' => PARAM_TEXT],
             'overwriteheaders' => ['type' => PARAM_BOOL],
+            'continueonerror' => ['type' => PARAM_BOOL],
             'delimiter' => ['type' => PARAM_TEXT],
         ];
     }
@@ -61,9 +62,11 @@ public function get_iterator(): iterator {
      */
     public function csv_contents_generator() {
         $maxlinelength = 1000;
-        $config = $this->get_variables()->get('config');
+        $variables = $this->get_variables();
+        $config = $variables->get('config');
         $strheaders = $config->headers;
         $overwriteheaders = !empty($config->overwriteheaders);
+        $continueonerror = !empty($config->continueonerror);
         $delimiter = $config->delimiter ?: self::DEFAULT_DELIMETER;
         $path = $this->enginestep->engine->resolve_path($config->path);
 
@@ -93,10 +96,30 @@ public function csv_contents_generator() {
             $headers = str_getcsv($strheaders, $delimiter);
             $numheaders = count($headers);
             $rownumber = 1; // First row is always headers.
+            $errors = ['header_field_count_mismatch' => 0];
             while (($data = fgetcsv($handle, $maxlinelength, $delimiter)) !== false) {
                 $rownumber++;
                 $numfields = count($data);
                 if ($numfields !== $numheaders) {
+                    // Continue on (parse) error.
+                    if ($continueonerror) {
+                        $errors['header_field_count_mismatch'] += 1;
+                        $this->log->error(
+                            get_string('reader_csv:header_field_count_mismatch', 'tool_dataflows', (object) [
+                                'numfields' => $numfields,
+                                'numheaders' => $numheaders,
+                                'rownumber' => $rownumber,
+                            ]),
+                            [
+                                'fields' => $data,
+                                'headers' => $headers,
+                            ]
+                        );
+
+                        continue;
+                    }
+
+                    // Throw exception on error.
                     throw new \moodle_exception('reader_csv:header_field_count_mismatch', 'tool_dataflows', '', (object) [
                         'numfields' => $numfields,
                         'numheaders' => $numheaders,
@@ -112,6 +135,8 @@ public function csv_contents_generator() {
         } finally {
             fclose($handle);
         }
+
+        $variables->set('errors', (object) $errors);
     }
 
     /**
@@ -153,6 +178,10 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
         $mform->hideIf('config_overwriteheaders', 'config_headers', 'eq', '');
         $mform->disabledIf('config_overwriteheaders', 'config_headers', 'eq', '');
 
+        // Used when we want to replace the headers (or overwrite them) using the ones we supplied instead. Defaults to off.
+        $mform->addElement('checkbox', 'config_continueonerror', get_string('reader_csv:continueonerror', 'tool_dataflows'),
+                get_string('reader_csv:continueonerror_help', 'tool_dataflows'));
+
         // Delimiter.
         $mform->addElement(
             'text',

diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php
@@ -373,6 +373,8 @@
 $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.';
 $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers';
 $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.';
+$string['reader_csv:continueonerror'] = 'Continue on parsing errors';
+$string['reader_csv:continueonerror_help'] = 'If checked, the step will continue reading the next row of data if there are parsing errors on the current.';
 $string['reader_csv:header_field_count_mismatch'] = 'Row #{$a->rownumber}: Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})';
 
 // Reader JSON.