From 6b789128ef76233dfde838263a51ae8749226b15 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sun, 16 Jun 2024 15:55:51 +0200 Subject: [PATCH] WIP: linting progresses --- .pylintrc | 5 ++ import-generator.py => import_generator.py | 0 src/anz_rosetta_csv/anz_rosetta_csv.py | 52 ++++++------------- src/anz_rosetta_csv/import_sheet_generator.py | 8 ++- .../provenance_csv_handler_class.py | 14 ++--- src/anz_rosetta_csv/rosetta_csv_generator.py | 29 ++++++----- .../rosetta_csv_sections_class.py | 6 +-- tests/test_import_generator.py | 3 +- 8 files changed, 55 insertions(+), 62 deletions(-) rename import-generator.py => import_generator.py (100%) diff --git a/.pylintrc b/.pylintrc index b6f1e79..527a942 100644 --- a/.pylintrc +++ b/.pylintrc @@ -23,6 +23,9 @@ ignore-patterns= ^(.+).html, ^(.+).htm, ^(.+).svg, + ^(.+).cfg, + ^(.+).prov, + ^(.+).notes, ^\., ignore-paths= @@ -33,3 +36,5 @@ ignore-paths= [MESSAGES CONTROL] disable = + C0301, # line too long. + R0903, # too few public methods. diff --git a/import-generator.py b/import_generator.py similarity index 100% rename from import-generator.py rename to import_generator.py diff --git a/src/anz_rosetta_csv/anz_rosetta_csv.py b/src/anz_rosetta_csv/anz_rosetta_csv.py index 9139e2c..f75c4fc 100644 --- a/src/anz_rosetta_csv/anz_rosetta_csv.py +++ b/src/anz_rosetta_csv/anz_rosetta_csv.py @@ -27,36 +27,12 @@ logging.Formatter.converter = time.gmtime -def createImportOverview(droidcsv, configfile): - createoverview = ImportOverviewGenerator(droidcsv, configfile) - createoverview.createOverviewSheet() - - -def importsheetDROIDmapping(droidcsv, importschema, configfile): - importgenerator = ImportSheetGenerator(droidcsv, importschema, configfile) - importgenerator.droid2archwayimport() - - -def exportsheetRosettamapping( - droidcsv, exportsheet, rosettaschema, configfile, provenance -): - csvgen = RosettaCSVGenerator( - droidcsv, exportsheet, rosettaschema, configfile, provenance - ) - res = csvgen.export2rosettacsv() - print(res) - - def main(): - # Usage: --csv [droid report] - # Handle command line arguments for the script + """Primary entry point for this script.""" + parser = argparse.ArgumentParser( description="Generate Archway Import Sheet and Rosetta Ingest CSV from DROID CSV Reports." ) - - # TODO: Consider optional and mandatory elements... behaviour might change depending on output... - # other options droid csv and rosetta schema - # NOTE: class on its own might be used to create a blank import csv with just static options parser.add_argument( "--csv", help="Single DROID CSV to read.", default=False, required=False ) @@ -89,12 +65,9 @@ def main(): if len(sys.argv) == 1: parser.print_help() - sys.exit(1) + sys.exit(0) - # Parse arguments into namespace object to reference later in the script - global args args = parser.parse_args() - if args.args: config = ConfigParser.RawConfigParser() config.read(args.args) @@ -108,13 +81,20 @@ def main(): args.exp = config.get("arguments", "listcontrol") args.pro = config.get("arguments", "provenance") - # creating an ingest sheet for Rosetta... if args.csv and args.exp and args.ros and args.cfg: - exportsheetRosettamapping(args.csv, args.exp, args.ros, args.cfg, args.pro) - # we're not doing anything sensible... - else: - parser.print_help() - sys.exit(1) + csvgen = RosettaCSVGenerator( + droidcsv=args.csv, + exportsheet=args.exp, + rosettaschema=args.ros, + configfile=args.cfg, + provenance=args.pro, + ) + res = csvgen.export2rosettacsv() + print(res) + sys.exit() + + parser.print_help() + sys.exit() if __name__ == "__main__": diff --git a/src/anz_rosetta_csv/import_sheet_generator.py b/src/anz_rosetta_csv/import_sheet_generator.py index 308079c..c0c765a 100644 --- a/src/anz_rosetta_csv/import_sheet_generator.py +++ b/src/anz_rosetta_csv/import_sheet_generator.py @@ -1,9 +1,13 @@ -#!/usr/local/bin/python -# -*- coding: utf-8 -*- +"""Functions shared with the Collections Import Sheet Generator.""" + +# pylint: disable=R0903 class ImportSheetGenerator: + """Import Sheet Generator Class.""" + def get_title(self, title): + """Return a title from a filename string.""" return title.rsplit(".", 1)[ 0 ].rstrip() # split once at full-stop (assumptuon 'ext' follows) diff --git a/src/anz_rosetta_csv/provenance_csv_handler_class.py b/src/anz_rosetta_csv/provenance_csv_handler_class.py index 236edcb..8caa114 100644 --- a/src/anz_rosetta_csv/provenance_csv_handler_class.py +++ b/src/anz_rosetta_csv/provenance_csv_handler_class.py @@ -4,21 +4,23 @@ from os.path import exists try: - from droid_csv_handler_class import * + from droid_csv_handler_class import GenericCSVHandler except ModuleNotFoundError: try: - from src.anz_rosetta_csv.droid_csv_handler_class import * + from src.anz_rosetta_csv.droid_csv_handler_class import GenericCSVHandler except ModuleNotFoundError: - from anz_rosetta_csv.droid_csv_handler_class import * + from anz_rosetta_csv.droid_csv_handler_class import GenericCSVHandler logger = logging.getLogger(__name__) -class provenanceCSVHandler: - # TODO: Better error handling? Format error handling? Validation? +class ProvenanceCSVHandler: + """Provenance CSV handler class.""" + provheaders = ["RECORDNUMBER", "NOTEDATE", "NOTETEXT"] - def readProvenanceCSV(self, provcsvname): + def read_provenance_csv(self, provcsvname): + """Read provenance CSV and return it to the caller.""" exportlist = None if not exists(provcsvname): logger.error( diff --git a/src/anz_rosetta_csv/rosetta_csv_generator.py b/src/anz_rosetta_csv/rosetta_csv_generator.py index c42249a..36464e4 100644 --- a/src/anz_rosetta_csv/rosetta_csv_generator.py +++ b/src/anz_rosetta_csv/rosetta_csv_generator.py @@ -9,20 +9,22 @@ import json_table_schema from droid_csv_handler_class import * from import_sheet_generator import ImportSheetGenerator - from provenance_csv_handler_class import * + from provenance_csv_handler_class import ProvenanceCSVHandler from rosetta_csv_sections_class import RosettaCSVSections except ModuleNotFoundError: try: from src.anz_rosetta_csv.droid_csv_handler_class import * from src.anz_rosetta_csv.import_sheet_generator import ImportSheetGenerator from src.anz_rosetta_csv.json_table_schema import json_table_schema - from src.anz_rosetta_csv.provenance_csv_handler_class import * + from src.anz_rosetta_csv.provenance_csv_handler_class import ( + ProvenanceCSVHandler, + ) from src.anz_rosetta_csv.rosetta_csv_sections_class import RosettaCSVSections except ModuleNotFoundError: from anz_rosetta_csv.droid_csv_handler_class import * from anz_rosetta_csv.import_sheet_generator import ImportSheetGenerator from anz_rosetta_csv.json_table_schema import json_table_schema - from anz_rosetta_csv.provenance_csv_handler_class import * + from anz_rosetta_csv.provenance_csv_handler_class import ProvenanceCSVHandler from anz_rosetta_csv.rosetta_csv_sections_class import RosettaCSVSections logger = logging.getLogger(__name__) @@ -117,8 +119,10 @@ def add_csv_value(self, value): return field def readRosettaSchema(self): - f = open(self.rosettaschema, "rb") - importschemajson = f.read() + """Read the Rosetta Schema File.""" + importschemajson = None + with open(self.rosettaschema, "r", encoding="utf-8") as rosetta_schema: + importschemajson = rosetta_schema.read() importschema = json_table_schema.JSONTableSchema(importschemajson) importschemadict = importschema.as_dict() @@ -128,7 +132,6 @@ def readRosettaSchema(self): importschemaheader + "\n" ) # TODO: Add newline in JSON Handler class? self.rosettacsvdict = importschemadict["fields"] - f.close() def createcolumns(self, columno): columns = "" @@ -223,10 +226,6 @@ def csvstringoutput(self, csvlist): csvrows = csvrows + "".join(SIPROW).rstrip(",") + "\n" - # write utf-8 BOM - # NOTE: Don't write UTF-8 BOM... Rosetta doesn't like this. - # sys.stdout.write(u'\uFEFF'.encode('utf-8')) - for sectionrows in csvlist: rowdata = "" for sectionrow in sectionrows: @@ -408,8 +407,8 @@ def create_rosetta_csv(self): return self.csvstringoutput(fields) - # TODO: unit tests for this... def listduplicates(self): + """List duplicates discovered running this script.""" seen = [] dupe = [] for row in self.droidlist: @@ -420,8 +419,10 @@ def listduplicates(self): cs = row["SHA1_HASH"] elif "SHA256_HASH" in row: cs = row["SHA256_HASH"] + elif "SHA512_HASH" in row: + cs = row["SHA512_HASH"] else: - sys.stderr.write("No hash available to use in DROID export.\n") + logging.error("no hash available to use in DROID export") sys.exit(1) if cs not in seen: seen.append(cs) @@ -449,8 +450,8 @@ def export2rosettacsv(self): # self.readRosettaSchema() #NOTE: Moved to constructor... TODO: Refactor if self.prov is True: - provhandler = provenanceCSVHandler() - self.provlist = provhandler.readProvenanceCSV(self.provfile) + provhandler = ProvenanceCSVHandler() + self.provlist = provhandler.read_provenance_csv(self.provfile) if self.provlist is None: self.prov = False diff --git a/src/anz_rosetta_csv/rosetta_csv_sections_class.py b/src/anz_rosetta_csv/rosetta_csv_sections_class.py index 632ef5c..dd366ce 100644 --- a/src/anz_rosetta_csv/rosetta_csv_sections_class.py +++ b/src/anz_rosetta_csv/rosetta_csv_sections_class.py @@ -2,6 +2,7 @@ import configparser as ConfigParser import logging +import sys logger = logging.getLogger(__name__) @@ -30,6 +31,5 @@ def __init__(self, configfile): sectiondict[section] = fieldlist.split(",") self.sections.append(sectiondict) else: - sys.stdout.write("Error reading fields from config file, exiting...") - sys.exit(1) # poor-form exiting from a child class? - break + logging.error("error reading fields from config file, exiting...") + sys.exit(1) diff --git a/tests/test_import_generator.py b/tests/test_import_generator.py index bb8a03f..0e26dfd 100644 --- a/tests/test_import_generator.py +++ b/tests/test_import_generator.py @@ -1,5 +1,7 @@ """Archives NZ import generator tests.""" +# pylint: disable=C0103 + from typing import Final from src.anz_rosetta_csv.rosetta_csv_generator import RosettaCSVGenerator @@ -639,7 +641,6 @@ def test_duplicates(tmp_path): schema_file = tmp_dir / "schema.json" droid_report = tmp_dir / "droid.csv" list_control_file = tmp_dir / "lc.csv" - tmp_dir / "prov.notes" config_file.write_text(dupe_config.strip().lstrip()) schema_file.write_text(schema.strip().lstrip())