skip rows

Author: radbrt

tap_sftp/singer_encodings/csv_handler.py

@@ -31,6 +31,9 @@ def get_row_iterator(iterable, options=None):
     which can be used to yield CSV rows."""
     options = options or {}
 
+    for i in range(options.get('skip_rows', 0)):
+        iterable.__next__()
+
     # Replace any NULL bytes in the line given to the DictReader
     reader = csv.DictReader(
         io.TextIOWrapper(iterable, encoding=options.get('encoding', 'utf-8')),

tap_sftp/singer_encodings/json_schema.py

@@ -42,7 +42,8 @@ def sample_file(conn, table_spec, f, sample_rate, max_records, config):
             'delimiter': table_spec.get('delimiter', ','),
             'file_name': f['filepath'],
             'encoding': table_spec.get('encoding', 'utf-8'),
-            'sanitize_header': table_spec.get('sanitize_header', False)}
+            'sanitize_header': table_spec.get('sanitize_header', False),
+            'skip_rows': table_spec.get('skip_rows', 0)}
 
     readers = csv_handler.get_row_iterators(file_handle, options=opts, infer_compression=True)
 

tap_sftp/sync.py

@@ -64,7 +64,8 @@ def sync_file(sftp_file_spec, stream, table_spec, config, sftp_client):
             'delimiter': table_spec.get('delimiter', ','),
             'file_name': sftp_file_spec['filepath'],
             'encoding': table_spec.get('encoding', 'utf-8'),
-            'sanitize_header': table_spec.get('sanitize_header', False)}
+            'sanitize_header': table_spec.get('sanitize_header', False),
+            'clean_colnames': table_spec.get('clean_colnames', False)}
 
     readers = csv_handler.get_row_iterators(file_handle, options=opts, infer_compression=True)
 

tests/configuration/skip_rows_file.csv

@@ -0,0 +1,3 @@
+# Skip reading this row
+id,col1
+data1,data2

tests/tox_tests/test_med_00_parse_csv.py

@@ -29,4 +29,19 @@ def test_no_sanitized_headers():
 
         parser = csv_handler.get_row_iterator(file, options=options)
 
-    assert parser.fieldnames == ['id', 'Col($2)']
+    assert parser.fieldnames == ['id', 'Col($2)']
+
+
+def test_skip_header():
+    """Test the parser."""
+    options = {
+        'delimiter': ',',
+        'key_properties': ['id'],
+        'skip_rows': 1,
+    }
+
+    with open(get_sample_file_path('skip_rows_file.csv'), 'rb') as file:
+
+        parser = csv_handler.get_row_iterator(file, options=options)
+
+    assert parser.fieldnames == ['id', 'col1']