WIP: linting progresses

Author: ross-spencer

.pylintrc

@@ -23,6 +23,9 @@ ignore-patterns=
     ^(.+).html,
     ^(.+).htm,
     ^(.+).svg,
+    ^(.+).cfg,
+    ^(.+).prov,
+    ^(.+).notes,
     ^\.,
 
 ignore-paths=
@@ -33,3 +36,5 @@ ignore-paths=
 [MESSAGES CONTROL]
 
 disable =
+    C0301,      # line too long.
+    R0903,      # too few public methods.

import-generator.py renamed to import_generator.py

@@ -27,36 +27,12 @@
 logging.Formatter.converter = time.gmtime
 
 
-def createImportOverview(droidcsv, configfile):
-    createoverview = ImportOverviewGenerator(droidcsv, configfile)
-    createoverview.createOverviewSheet()
-
-
-def importsheetDROIDmapping(droidcsv, importschema, configfile):
-    importgenerator = ImportSheetGenerator(droidcsv, importschema, configfile)
-    importgenerator.droid2archwayimport()
-
-
-def exportsheetRosettamapping(
-    droidcsv, exportsheet, rosettaschema, configfile, provenance
-):
-    csvgen = RosettaCSVGenerator(
-        droidcsv, exportsheet, rosettaschema, configfile, provenance
-    )
-    res = csvgen.export2rosettacsv()
-    print(res)
-
-
 def main():
-    # 	Usage: 	--csv [droid report]
-    # 	Handle command line arguments for the script
+    """Primary entry point for this script."""
+
     parser = argparse.ArgumentParser(
         description="Generate Archway Import Sheet and Rosetta Ingest CSV from DROID CSV Reports."
     )
-
-    # TODO: Consider optional and mandatory elements... behaviour might change depending on output...
-    # other options droid csv and rosetta schema
-    # NOTE: class on its own might be used to create a blank import csv with just static options
     parser.add_argument(
         "--csv", help="Single DROID CSV to read.", default=False, required=False
     )
@@ -89,12 +65,9 @@ def main():
 
     if len(sys.argv) == 1:
         parser.print_help()
-        sys.exit(1)
+        sys.exit(0)
 
-    # 	Parse arguments into namespace object to reference later in the script
-    global args
     args = parser.parse_args()
-
     if args.args:
         config = ConfigParser.RawConfigParser()
         config.read(args.args)
@@ -108,13 +81,20 @@ def main():
             args.exp = config.get("arguments", "listcontrol")
             args.pro = config.get("arguments", "provenance")
 
-    # creating an ingest sheet for Rosetta...
     if args.csv and args.exp and args.ros and args.cfg:
-        exportsheetRosettamapping(args.csv, args.exp, args.ros, args.cfg, args.pro)
-    # we're not doing anything sensible...
-    else:
-        parser.print_help()
-        sys.exit(1)
+        csvgen = RosettaCSVGenerator(
+            droidcsv=args.csv,
+            exportsheet=args.exp,
+            rosettaschema=args.ros,
+            configfile=args.cfg,
+            provenance=args.pro,
+        )
+        res = csvgen.export2rosettacsv()
+        print(res)
+        sys.exit()
+
+    parser.print_help()
+    sys.exit()
 
 
 if __name__ == "__main__":

src/anz_rosetta_csv/anz_rosetta_csv.py

@@ -1,9 +1,13 @@
-#!/usr/local/bin/python
-# -*- coding: utf-8 -*-
+"""Functions shared with the Collections Import Sheet Generator."""
+
+# pylint: disable=R0903
 
 
 class ImportSheetGenerator:
+    """Import Sheet Generator Class."""
+
     def get_title(self, title):
+        """Return a title from a filename string."""
         return title.rsplit(".", 1)[
             0
         ].rstrip()  # split once at full-stop (assumptuon 'ext' follows)

src/anz_rosetta_csv/import_sheet_generator.py

@@ -4,21 +4,23 @@
 from os.path import exists
 
 try:
-    from droid_csv_handler_class import *
+    from droid_csv_handler_class import GenericCSVHandler
 except ModuleNotFoundError:
     try:
-        from src.anz_rosetta_csv.droid_csv_handler_class import *
+        from src.anz_rosetta_csv.droid_csv_handler_class import GenericCSVHandler
     except ModuleNotFoundError:
-        from anz_rosetta_csv.droid_csv_handler_class import *
+        from anz_rosetta_csv.droid_csv_handler_class import GenericCSVHandler
 
 logger = logging.getLogger(__name__)
 
 
-class provenanceCSVHandler:
-    # TODO: Better error handling? Format error handling? Validation?
+class ProvenanceCSVHandler:
+    """Provenance CSV handler class."""
+
     provheaders = ["RECORDNUMBER", "NOTEDATE", "NOTETEXT"]
 
-    def readProvenanceCSV(self, provcsvname):
+    def read_provenance_csv(self, provcsvname):
+        """Read provenance CSV and return it to the caller."""
         exportlist = None
         if not exists(provcsvname):
             logger.error(

src/anz_rosetta_csv/provenance_csv_handler_class.py

@@ -9,20 +9,22 @@
     import json_table_schema
     from droid_csv_handler_class import *
     from import_sheet_generator import ImportSheetGenerator
-    from provenance_csv_handler_class import *
+    from provenance_csv_handler_class import ProvenanceCSVHandler
     from rosetta_csv_sections_class import RosettaCSVSections
 except ModuleNotFoundError:
     try:
         from src.anz_rosetta_csv.droid_csv_handler_class import *
         from src.anz_rosetta_csv.import_sheet_generator import ImportSheetGenerator
         from src.anz_rosetta_csv.json_table_schema import json_table_schema
-        from src.anz_rosetta_csv.provenance_csv_handler_class import *
+        from src.anz_rosetta_csv.provenance_csv_handler_class import (
+            ProvenanceCSVHandler,
+        )
         from src.anz_rosetta_csv.rosetta_csv_sections_class import RosettaCSVSections
     except ModuleNotFoundError:
         from anz_rosetta_csv.droid_csv_handler_class import *
         from anz_rosetta_csv.import_sheet_generator import ImportSheetGenerator
         from anz_rosetta_csv.json_table_schema import json_table_schema
-        from anz_rosetta_csv.provenance_csv_handler_class import *
+        from anz_rosetta_csv.provenance_csv_handler_class import ProvenanceCSVHandler
         from anz_rosetta_csv.rosetta_csv_sections_class import RosettaCSVSections
 
 logger = logging.getLogger(__name__)
@@ -117,8 +119,10 @@ def add_csv_value(self, value):
         return field
 
     def readRosettaSchema(self):
-        f = open(self.rosettaschema, "rb")
-        importschemajson = f.read()
+        """Read the Rosetta Schema File."""
+        importschemajson = None
+        with open(self.rosettaschema, "r", encoding="utf-8") as rosetta_schema:
+            importschemajson = rosetta_schema.read()
         importschema = json_table_schema.JSONTableSchema(importschemajson)
 
         importschemadict = importschema.as_dict()
@@ -128,7 +132,6 @@ def readRosettaSchema(self):
             importschemaheader + "\n"
         )  # TODO: Add newline in JSON Handler class?
         self.rosettacsvdict = importschemadict["fields"]
-        f.close()
 
     def createcolumns(self, columno):
         columns = ""
@@ -223,10 +226,6 @@ def csvstringoutput(self, csvlist):
 
         csvrows = csvrows + "".join(SIPROW).rstrip(",") + "\n"
 
-        # write utf-8 BOM
-        # NOTE: Don't write UTF-8 BOM... Rosetta doesn't like this.
-        # sys.stdout.write(u'\uFEFF'.encode('utf-8'))
-
         for sectionrows in csvlist:
             rowdata = ""
             for sectionrow in sectionrows:
@@ -408,8 +407,8 @@ def create_rosetta_csv(self):
 
         return self.csvstringoutput(fields)
 
-    # TODO: unit tests for this...
     def listduplicates(self):
+        """List duplicates discovered running this script."""
         seen = []
         dupe = []
         for row in self.droidlist:
@@ -420,8 +419,10 @@ def listduplicates(self):
                 cs = row["SHA1_HASH"]
             elif "SHA256_HASH" in row:
                 cs = row["SHA256_HASH"]
+            elif "SHA512_HASH" in row:
+                cs = row["SHA512_HASH"]
             else:
-                sys.stderr.write("No hash available to use in DROID export.\n")
+                logging.error("no hash available to use in DROID export")
                 sys.exit(1)
             if cs not in seen:
                 seen.append(cs)
@@ -449,8 +450,8 @@ def export2rosettacsv(self):
             # self.readRosettaSchema()  #NOTE: Moved to constructor... TODO: Refactor
 
             if self.prov is True:
-                provhandler = provenanceCSVHandler()
-                self.provlist = provhandler.readProvenanceCSV(self.provfile)
+                provhandler = ProvenanceCSVHandler()
+                self.provlist = provhandler.read_provenance_csv(self.provfile)
                 if self.provlist is None:
                     self.prov = False
 

src/anz_rosetta_csv/rosetta_csv_generator.py

@@ -2,6 +2,7 @@
 
 import configparser as ConfigParser
 import logging
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +31,5 @@ def __init__(self, configfile):
                 sectiondict[section] = fieldlist.split(",")
                 self.sections.append(sectiondict)
             else:
-                sys.stdout.write("Error reading fields from config file, exiting...")
-                sys.exit(1)  # poor-form exiting from a child class?
-                break
+                logging.error("error reading fields from config file, exiting...")
+                sys.exit(1)

src/anz_rosetta_csv/rosetta_csv_sections_class.py

@@ -1,5 +1,7 @@
 """Archives NZ import generator tests."""
 
+# pylint: disable=C0103
+
 from typing import Final
 
 from src.anz_rosetta_csv.rosetta_csv_generator import RosettaCSVGenerator
@@ -639,7 +641,6 @@ def test_duplicates(tmp_path):
     schema_file = tmp_dir / "schema.json"
     droid_report = tmp_dir / "droid.csv"
     list_control_file = tmp_dir / "lc.csv"
-    tmp_dir / "prov.notes"
 
     config_file.write_text(dupe_config.strip().lstrip())
     schema_file.write_text(schema.strip().lstrip())