diff --git a/accounting/aggregations/create_report.py b/accounting/aggregations/create_report.py index de9d261..a06e8d2 100644 --- a/accounting/aggregations/create_report.py +++ b/accounting/aggregations/create_report.py @@ -2,23 +2,15 @@ # parses cmd line args and handles outputs import elasticsearch -from elasticsearch_dsl import Search, Q, A, connections import argparse import sys import json -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path -from pprint import pprint from query import run_query -from functions import send_email -from report_helpers import ( - Aggregation, - add_runtime_script, - get_percent_bucket_script, - table, print_error, - generate_csv -) +from functions import send_email +from report_helpers import table, generate_csv OUTPUT_ARGS = { "--print-table" : {"action" : "store_true", "help" : "prints a CLI table, NOTE: pipe into 'less -S'"}, diff --git a/accounting/aggregations/functions.py b/accounting/aggregations/functions.py index 8799a35..1a0b11e 100644 --- a/accounting/aggregations/functions.py +++ b/accounting/aggregations/functions.py @@ -15,9 +15,15 @@ from pathlib import Path try: - import htcondor -except ModuleNotFoundError: - htcondor = None + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + htcondor = None + from dns.resolver import query as dns_query diff --git a/accounting/aggregations/missing_ospool_projects.py b/accounting/aggregations/missing_ospool_projects.py index f8d7723..0e820e4 100644 --- a/accounting/aggregations/missing_ospool_projects.py +++ b/accounting/aggregations/missing_ospool_projects.py @@ -1,5 +1,4 @@ import sys -import time import json import argparse import importlib diff --git a/accounting/aggregations/query.py b/accounting/aggregations/query.py index c3b7047..8686336 100755 --- a/accounting/aggregations/query.py +++ b/accounting/aggregations/query.py @@ -1,15 +1,10 @@ import elasticsearch import argparse -import sys -import json -from functions import send_email -from elasticsearch_dsl import Search, Q, A, connections, response -from datetime import datetime, timedelta -from collections import namedtuple +from elasticsearch_dsl import Search, Q, A, response +from datetime import datetime from operator import itemgetter -from pprint import pprint -from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script, table, print_error +from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script # lists to hold aggregation objects ROWS_AGGS = [] diff --git a/accounting/filters/BaseFilter.py b/accounting/filters/BaseFilter.py index 386dc30..5b40956 100644 --- a/accounting/filters/BaseFilter.py +++ b/accounting/filters/BaseFilter.py @@ -1,5 +1,4 @@ import logging -import statistics as stats from collections import defaultdict from functools import partial from operator import itemgetter diff --git a/accounting/filters/ChtcScheddCpuFilter.py b/accounting/filters/ChtcScheddCpuFilter.py index 6845449..ee9903b 100644 --- a/accounting/filters/ChtcScheddCpuFilter.py +++ b/accounting/filters/ChtcScheddCpuFilter.py @@ -1,5 +1,4 @@ import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/ChtcScheddCpuMonthlyFilter.py b/accounting/filters/ChtcScheddCpuMonthlyFilter.py index 2774598..36d7c4b 100644 --- a/accounting/filters/ChtcScheddCpuMonthlyFilter.py +++ b/accounting/filters/ChtcScheddCpuMonthlyFilter.py @@ -1,9 +1,6 @@ - -import statistics as stats from collections import defaultdict from operator import itemgetter from ast import literal_eval -import elasticsearch.helpers from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/ChtcScheddCpuOspoolFilter.py b/accounting/filters/ChtcScheddCpuOspoolFilter.py index e0fd7ed..09b9530 100644 --- a/accounting/filters/ChtcScheddCpuOspoolFilter.py +++ b/accounting/filters/ChtcScheddCpuOspoolFilter.py @@ -1,6 +1,6 @@ import re +import sys import pickle -import htcondor import statistics as stats from datetime import date from pathlib import Path @@ -8,6 +8,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_job_units +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py b/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py index 092eec4..94c881b 100644 --- a/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py +++ b/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py @@ -1,16 +1,23 @@ import re +import sys import pickle -import htcondor -import statistics as stats from collections import defaultdict from operator import itemgetter from ast import literal_eval -import elasticsearch.helpers from functools import lru_cache from pathlib import Path from .BaseFilter import BaseFilter from accounting.functions import get_job_units +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + MAX_INT = 2**62 DEFAULT_COLUMNS = { diff --git a/accounting/filters/ChtcScheddCpuRemovedFilter.py b/accounting/filters/ChtcScheddCpuRemovedFilter.py index 4ee0a44..9f4702d 100644 --- a/accounting/filters/ChtcScheddCpuRemovedFilter.py +++ b/accounting/filters/ChtcScheddCpuRemovedFilter.py @@ -1,6 +1,4 @@ -import htcondor import statistics as stats -from pathlib import Path from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddDSIGpuFilter.py b/accounting/filters/ChtcScheddDSIGpuFilter.py index cbb655c..b48b8ad 100644 --- a/accounting/filters/ChtcScheddDSIGpuFilter.py +++ b/accounting/filters/ChtcScheddDSIGpuFilter.py @@ -1,6 +1,3 @@ - -import statistics as stats -from pathlib import Path from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddGpuFilter.py b/accounting/filters/ChtcScheddGpuFilter.py index 274f90a..0d014a3 100644 --- a/accounting/filters/ChtcScheddGpuFilter.py +++ b/accounting/filters/ChtcScheddGpuFilter.py @@ -1,6 +1,4 @@ - import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddJobDistroFilter.py b/accounting/filters/ChtcScheddJobDistroFilter.py index 23740b7..ccec136 100644 --- a/accounting/filters/ChtcScheddJobDistroFilter.py +++ b/accounting/filters/ChtcScheddJobDistroFilter.py @@ -1,8 +1,3 @@ - -import htcondor -import pickle -from pathlib import Path -from elasticsearch import Elasticsearch import elasticsearch.helpers from .BaseFilter import BaseFilter from functools import lru_cache diff --git a/accounting/filters/IgwnScheddCpuFilter.py b/accounting/filters/IgwnScheddCpuFilter.py index f3a1be0..ccf0f39 100644 --- a/accounting/filters/IgwnScheddCpuFilter.py +++ b/accounting/filters/IgwnScheddCpuFilter.py @@ -1,6 +1,4 @@ import statistics as stats -from pathlib import Path -from ast import literal_eval from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/IgwnScheddCpuMonthlyFilter.py b/accounting/filters/IgwnScheddCpuMonthlyFilter.py index 6c19007..5619425 100644 --- a/accounting/filters/IgwnScheddCpuMonthlyFilter.py +++ b/accounting/filters/IgwnScheddCpuMonthlyFilter.py @@ -1,4 +1,3 @@ -from pathlib import Path from collections import defaultdict from operator import itemgetter from .BaseFilter import BaseFilter diff --git a/accounting/filters/OsgScheddCpuFilter.py b/accounting/filters/OsgScheddCpuFilter.py index a711617..1b12284 100644 --- a/accounting/filters/OsgScheddCpuFilter.py +++ b/accounting/filters/OsgScheddCpuFilter.py @@ -1,6 +1,5 @@ - import re -import htcondor +import sys import pickle import statistics as stats from datetime import date @@ -8,6 +7,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_job_units, get_topology_project_data, get_topology_resource_data, get_institution_database +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 5: "% Shadw w/o Start", diff --git a/accounting/filters/OsgScheddCpuHeldFilter.py b/accounting/filters/OsgScheddCpuHeldFilter.py index b634d2a..ae0305c 100644 --- a/accounting/filters/OsgScheddCpuHeldFilter.py +++ b/accounting/filters/OsgScheddCpuHeldFilter.py @@ -1,11 +1,19 @@ - import re -import htcondor +import sys import pickle import statistics as stats from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise HOLD_REASONS = [ "Unspecified", diff --git a/accounting/filters/OsgScheddCpuMonthlyFilter.py b/accounting/filters/OsgScheddCpuMonthlyFilter.py index 8ba7718..97ec32e 100644 --- a/accounting/filters/OsgScheddCpuMonthlyFilter.py +++ b/accounting/filters/OsgScheddCpuMonthlyFilter.py @@ -1,4 +1,3 @@ - import re import htcondor import pickle diff --git a/accounting/filters/OsgScheddCpuRemovedFilter.py b/accounting/filters/OsgScheddCpuRemovedFilter.py index 96096cc..2757a8f 100644 --- a/accounting/filters/OsgScheddCpuRemovedFilter.py +++ b/accounting/filters/OsgScheddCpuRemovedFilter.py @@ -1,11 +1,21 @@ import re -import htcondor +import sys import pickle import statistics as stats from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddCpuRetryFilter.py b/accounting/filters/OsgScheddCpuRetryFilter.py index 7f24f5f..c0ef45b 100644 --- a/accounting/filters/OsgScheddCpuRetryFilter.py +++ b/accounting/filters/OsgScheddCpuRetryFilter.py @@ -1,11 +1,21 @@ import re -import htcondor +import sys import pickle from pathlib import Path from .BaseFilter import BaseFilter from accounting.pull_hold_reasons import get_hold_reasons +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddGpuFilter.py b/accounting/filters/OsgScheddGpuFilter.py index 12468fe..ef23fc6 100644 --- a/accounting/filters/OsgScheddGpuFilter.py +++ b/accounting/filters/OsgScheddGpuFilter.py @@ -1,6 +1,6 @@ import re -import htcondor +import sys import pickle import statistics as stats from datetime import date @@ -8,6 +8,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_topology_resource_data, get_institution_database +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddJobDistroFilter.py b/accounting/filters/OsgScheddJobDistroFilter.py index 8ed7a8e..0dfb1a2 100644 --- a/accounting/filters/OsgScheddJobDistroFilter.py +++ b/accounting/filters/OsgScheddJobDistroFilter.py @@ -1,14 +1,23 @@ import re -import htcondor +import sys import pickle from pathlib import Path -from elasticsearch import Elasticsearch import elasticsearch.helpers from .BaseFilter import BaseFilter from functools import lru_cache from collections import defaultdict +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + OSG_CONNECT_APS = { "login04.osgconnect.net", diff --git a/accounting/filters/OsgScheddLongJobFilter.py b/accounting/filters/OsgScheddLongJobFilter.py index 92b6b9f..18ef4ba 100644 --- a/accounting/filters/OsgScheddLongJobFilter.py +++ b/accounting/filters/OsgScheddLongJobFilter.py @@ -1,10 +1,20 @@ import re -import htcondor +import sys import pickle from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 5 : "Project", diff --git a/accounting/filters/PathScheddCpuFilter.py b/accounting/filters/PathScheddCpuFilter.py index 350f8e0..fdcdf0b 100644 --- a/accounting/filters/PathScheddCpuFilter.py +++ b/accounting/filters/PathScheddCpuFilter.py @@ -1,5 +1,4 @@ import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter diff --git a/accounting/formatters/BaseFormatter.py b/accounting/formatters/BaseFormatter.py index 1740d52..38a6e6b 100644 --- a/accounting/formatters/BaseFormatter.py +++ b/accounting/formatters/BaseFormatter.py @@ -1,6 +1,6 @@ import csv -from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta +from collections import OrderedDict +from datetime import datetime from pathlib import Path diff --git a/accounting/formatters/ChtcScheddJobDistroFormatter.py b/accounting/formatters/ChtcScheddJobDistroFormatter.py index a372fba..3611549 100644 --- a/accounting/formatters/ChtcScheddJobDistroFormatter.py +++ b/accounting/formatters/ChtcScheddJobDistroFormatter.py @@ -1,8 +1,6 @@ import csv -from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path -from decimal import Decimal def break_chars(s): diff --git a/accounting/pull_hold_reasons.py b/accounting/pull_hold_reasons.py index 94de84e..86ed93e 100644 --- a/accounting/pull_hold_reasons.py +++ b/accounting/pull_hold_reasons.py @@ -1,7 +1,6 @@ import pickle import tempfile import os -import time import re from urllib.request import urlopen from pathlib import Path diff --git a/cache_collector_hosts.py b/cache_collector_hosts.py index b44ba27..da9dc88 100644 --- a/cache_collector_hosts.py +++ b/cache_collector_hosts.py @@ -1,8 +1,19 @@ import re -import htcondor +import sys import pickle from pathlib import Path +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + + CUSTOM_MAPPING = { "osg-login2.pace.gatech.edu": {"osg-login2.pace.gatech.edu"}, "ce1.opensciencegrid.org": {"cm-2.ospool.osg-htc.org", "cm-1.ospool.osg-htc.org"}, diff --git a/current/ospool_gpu_job_shapes.py b/current/ospool_gpu_job_shapes.py index bb912d4..4a22c8f 100644 --- a/current/ospool_gpu_job_shapes.py +++ b/current/ospool_gpu_job_shapes.py @@ -1,3 +1,4 @@ +import sys import math import time import argparse @@ -7,8 +8,17 @@ from typing import List, Tuple, Dict, Set, Union from pprint import pprint -import htcondor -import classad +try: + import htcondor2 as htcondor + import classad2 as classad +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + import classad + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise DEFAULT_COLLECTOR = "cm-1.ospool.osg-htc.org"