INF-3316 Fix htcondor import statements and remove unused imports

accounting/aggregations/create_report.py

@@ -2,23 +2,15 @@
 # parses cmd line args and handles outputs
 
 import elasticsearch
-from elasticsearch_dsl import Search, Q, A, connections
 import argparse
 import sys
 import json
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
-from pprint import pprint
 
 from query import run_query
-from functions import send_email 
-from report_helpers import (
-    Aggregation, 
-    add_runtime_script, 
-    get_percent_bucket_script, 
-    table, print_error, 
-    generate_csv
-)
+from functions import send_email
+from report_helpers import table, generate_csv
 
 OUTPUT_ARGS = {
     "--print-table" : {"action" : "store_true", "help" : "prints a CLI table, NOTE: pipe into 'less -S'"},

accounting/aggregations/functions.py

@@ -15,9 +15,15 @@
 from pathlib import Path
 
 try:
-    import htcondor
-except ModuleNotFoundError:
-    htcondor = None
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        htcondor = None
+
 from dns.resolver import query as dns_query
 
 

accounting/aggregations/missing_ospool_projects.py

@@ -1,5 +1,4 @@
 import sys
-import time
 import json
 import argparse
 import importlib

accounting/aggregations/query.py

@@ -1,15 +1,10 @@
 import elasticsearch
 import argparse
-import sys
-import json
-from functions import send_email
-from elasticsearch_dsl import Search, Q, A, connections, response
-from datetime import datetime, timedelta
-from collections import namedtuple
+from elasticsearch_dsl import Search, Q, A, response
+from datetime import datetime
 from operator import itemgetter
-from pprint import pprint
 
-from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script, table, print_error
+from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script
 
 # lists to hold aggregation objects
 ROWS_AGGS = []

accounting/filters/BaseFilter.py

@@ -1,5 +1,4 @@
 import logging
-import statistics as stats
 from collections import defaultdict
 from functools import partial
 from operator import itemgetter

accounting/filters/ChtcScheddCpuFilter.py

@@ -1,5 +1,4 @@
 import statistics as stats
-from pathlib import Path
 from ast import literal_eval
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units

accounting/filters/ChtcScheddCpuMonthlyFilter.py

@@ -1,9 +1,6 @@
-
-import statistics as stats
 from collections import defaultdict
 from operator import itemgetter
 from ast import literal_eval
-import elasticsearch.helpers
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units
 

accounting/filters/ChtcScheddCpuOspoolFilter.py

@@ -1,13 +1,23 @@
 import re
+import sys
 import pickle
-import htcondor
 import statistics as stats
 from datetime import date
 from pathlib import Path
 from ast import literal_eval
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
     10: "Num Uniq Job Ids",

accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py

@@ -1,16 +1,23 @@
 import re
+import sys
 import pickle
-import htcondor
-import statistics as stats
 from collections import defaultdict
 from operator import itemgetter
 from ast import literal_eval
-import elasticsearch.helpers
 from functools import lru_cache
 from pathlib import Path
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+
 MAX_INT = 2**62
 
 DEFAULT_COLUMNS = {

accounting/filters/ChtcScheddCpuRemovedFilter.py

@@ -1,6 +1,4 @@
-import htcondor
 import statistics as stats
-from pathlib import Path
 from .BaseFilter import BaseFilter
 
 

accounting/filters/ChtcScheddDSIGpuFilter.py

@@ -1,6 +1,3 @@
-
-import statistics as stats
-from pathlib import Path
 from .BaseFilter import BaseFilter
 
 

accounting/filters/ChtcScheddGpuFilter.py

@@ -1,6 +1,4 @@
-
 import statistics as stats
-from pathlib import Path
 from ast import literal_eval
 from .BaseFilter import BaseFilter
 

accounting/filters/ChtcScheddJobDistroFilter.py

@@ -1,8 +1,3 @@
-
-import htcondor
-import pickle
-from pathlib import Path
-from elasticsearch import Elasticsearch
 import elasticsearch.helpers
 from .BaseFilter import BaseFilter
 from functools import lru_cache

accounting/filters/IgwnScheddCpuFilter.py

@@ -1,6 +1,4 @@
 import statistics as stats
-from pathlib import Path
-from ast import literal_eval
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units
 

accounting/filters/IgwnScheddCpuMonthlyFilter.py

@@ -1,4 +1,3 @@
-from pathlib import Path
 from collections import defaultdict
 from operator import itemgetter
 from .BaseFilter import BaseFilter

accounting/filters/OsgScheddCpuFilter.py

@@ -1,13 +1,22 @@
-
 import re
-import htcondor
+import sys
 import pickle
 import statistics as stats
 from datetime import date
 from pathlib import Path
 from .BaseFilter import BaseFilter
 from accounting.functions import get_job_units, get_topology_project_data, get_topology_resource_data, get_institution_database
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
      5: "% Shadw w/o Start",

accounting/filters/OsgScheddCpuHeldFilter.py

@@ -1,11 +1,19 @@
-
 import re
-import htcondor
+import sys
 import pickle
 import statistics as stats
 from pathlib import Path
 from .BaseFilter import BaseFilter
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
 
 HOLD_REASONS = [
     "Unspecified",

accounting/filters/OsgScheddCpuMonthlyFilter.py

@@ -1,4 +1,3 @@
-
 import re
 import htcondor
 import pickle

accounting/filters/OsgScheddCpuRemovedFilter.py

@@ -1,11 +1,21 @@
 
 import re
-import htcondor
+import sys
 import pickle
 import statistics as stats
 from pathlib import Path
 from .BaseFilter import BaseFilter
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
     10: "Num Uniq Job Ids",

accounting/filters/OsgScheddCpuRetryFilter.py

@@ -1,11 +1,21 @@
 
 import re
-import htcondor
+import sys
 import pickle
 from pathlib import Path
 from .BaseFilter import BaseFilter
 from accounting.pull_hold_reasons import get_hold_reasons
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
     10: "Num Uniq Job Ids",

accounting/filters/OsgScheddGpuFilter.py

@@ -1,13 +1,23 @@
 
 import re
-import htcondor
+import sys
 import pickle
 import statistics as stats
 from datetime import date
 from pathlib import Path
 from .BaseFilter import BaseFilter
 from accounting.functions import get_topology_resource_data, get_institution_database
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
     10: "Num Uniq Job Ids",

accounting/filters/OsgScheddJobDistroFilter.py

@@ -1,14 +1,23 @@
 
 import re
-import htcondor
+import sys
 import pickle
 from pathlib import Path
-from elasticsearch import Elasticsearch
 import elasticsearch.helpers
 from .BaseFilter import BaseFilter
 from functools import lru_cache
 from collections import defaultdict
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 OSG_CONNECT_APS = {
     "login04.osgconnect.net",

accounting/filters/OsgScheddLongJobFilter.py

@@ -1,10 +1,20 @@
 
 import re
-import htcondor
+import sys
 import pickle
 from pathlib import Path
 from .BaseFilter import BaseFilter
 
+try:
+    import htcondor2 as htcondor
+except ImportError:
+    print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr)
+    try:
+        import htcondor
+    except ImportError:
+        print("Could not import htcondor", file=sys.stderr)
+        raise
+
 
 DEFAULT_COLUMNS = {
     5 : "Project",

accounting/filters/PathScheddCpuFilter.py

@@ -1,5 +1,4 @@
 import statistics as stats
-from pathlib import Path
 from ast import literal_eval
 from .BaseFilter import BaseFilter
 

accounting/formatters/BaseFormatter.py

@@ -1,6 +1,6 @@
 import csv
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
+from collections import OrderedDict
+from datetime import datetime
 from pathlib import Path
 
 

accounting/formatters/ChtcScheddJobDistroFormatter.py

@@ -1,8 +1,6 @@
 import csv
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
-from decimal import Decimal
 
 
 def break_chars(s):

accounting/pull_hold_reasons.py

@@ -1,7 +1,6 @@
 import pickle
 import tempfile
 import os
-import time
 import re
 from urllib.request import urlopen
 from pathlib import Path