From 8e3f15ff9479cef410d84145e934c1d2a7e46030 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 21:41:43 +0000 Subject: [PATCH 1/4] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.6.0) - [github.com/psf/black: 23.3.0 → 24.4.0](https://github.com/psf/black/compare/23.3.0...24.4.0) - [github.com/PyCQA/flake8: 6.0.0 → 7.0.0](https://github.com/PyCQA/flake8/compare/6.0.0...7.0.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45000c41..362aa937 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: check-docstring-first @@ -16,11 +16,11 @@ repos: - id: end-of-file-fixer - id: sort-simple-yaml - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.4.0 hooks: - id: black args: ['--line-length=119'] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.0.0 hooks: - id: flake8 From 8e16269e05237060a55f8841e020a1cd08bfafc6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 21:42:10 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/dasmon_app/dasmon_listener/listener_daemon.py | 1 + src/webmon_app/reporting/dasmon/legacy_status.py | 1 + src/webmon_app/reporting/dasmon/models.py | 1 + src/webmon_app/reporting/dasmon/view_util.py | 1 + src/webmon_app/reporting/pvmon/models.py | 1 + src/webmon_app/reporting/pvmon/views.py | 1 + src/webmon_app/reporting/reduction/view_util.py | 1 + src/webmon_app/reporting/report/models.py | 1 + src/webmon_app/reporting/report/view_util.py | 1 + src/webmon_app/reporting/reporting_app/settings/develop.py | 1 + src/webmon_app/reporting/reporting_app/settings/envtest.py | 1 + src/webmon_app/reporting/reporting_app/settings/prod.py | 1 + src/webmon_app/reporting/reporting_app/settings/unittest.py | 1 + src/webmon_app/reporting/reporting_app/view_util.py | 1 + src/webmon_app/reporting/reporting_app/wsgi.py | 1 + src/webmon_app/reporting/users/views.py | 1 + src/workflow_app/workflow/states.py | 1 + src/workflow_app/workflow/workflow_process.py | 1 + tests/simple_producer.py | 1 + tests/workflow_command.py | 1 + 20 files changed, 20 insertions(+) diff --git a/src/dasmon_app/dasmon_listener/listener_daemon.py b/src/dasmon_app/dasmon_listener/listener_daemon.py index f1d73ba8..0744ee97 100644 --- a/src/dasmon_app/dasmon_listener/listener_daemon.py +++ b/src/dasmon_app/dasmon_listener/listener_daemon.py @@ -1,6 +1,7 @@ """ DASMON listener daemon """ + import sys import argparse diff --git a/src/webmon_app/reporting/dasmon/legacy_status.py b/src/webmon_app/reporting/dasmon/legacy_status.py index cbba6f13..404f6a87 100644 --- a/src/webmon_app/reporting/dasmon/legacy_status.py +++ b/src/webmon_app/reporting/dasmon/legacy_status.py @@ -4,6 +4,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2015 Oak Ridge National Laboratory """ + import httplib2 import json import logging diff --git a/src/webmon_app/reporting/dasmon/models.py b/src/webmon_app/reporting/dasmon/models.py index 3c4ec1ac..b7302336 100644 --- a/src/webmon_app/reporting/dasmon/models.py +++ b/src/webmon_app/reporting/dasmon/models.py @@ -3,6 +3,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2015 Oak Ridge National Laboratory """ + from django.db import models from reporting.report.models import Instrument diff --git a/src/webmon_app/reporting/dasmon/view_util.py b/src/webmon_app/reporting/dasmon/view_util.py index b5e41df6..209481c8 100644 --- a/src/webmon_app/reporting/dasmon/view_util.py +++ b/src/webmon_app/reporting/dasmon/view_util.py @@ -4,6 +4,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2014 Oak Ridge National Laboratory """ + from reporting.report.models import Instrument, DataRun, WorkflowSummary from reporting.dasmon.models import ( Parameter, diff --git a/src/webmon_app/reporting/pvmon/models.py b/src/webmon_app/reporting/pvmon/models.py index 6e934736..b9665789 100644 --- a/src/webmon_app/reporting/pvmon/models.py +++ b/src/webmon_app/reporting/pvmon/models.py @@ -4,6 +4,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2014 Oak Ridge National Laboratory """ + from django.db import models from reporting.report.models import Instrument diff --git a/src/webmon_app/reporting/pvmon/views.py b/src/webmon_app/reporting/pvmon/views.py index c1fa8f80..a54f2526 100644 --- a/src/webmon_app/reporting/pvmon/views.py +++ b/src/webmon_app/reporting/pvmon/views.py @@ -1,6 +1,7 @@ """ Live PV monitoring """ + from django.http import HttpResponse from django.shortcuts import render, get_object_or_404 from django.urls import reverse diff --git a/src/webmon_app/reporting/reduction/view_util.py b/src/webmon_app/reporting/reduction/view_util.py index 84e7b228..01902e53 100644 --- a/src/webmon_app/reporting/reduction/view_util.py +++ b/src/webmon_app/reporting/reduction/view_util.py @@ -4,6 +4,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2014 Oak Ridge National Laboratory """ + from django.conf import settings from django.urls import reverse import logging diff --git a/src/webmon_app/reporting/report/models.py b/src/webmon_app/reporting/report/models.py index 283ae0cc..68828373 100644 --- a/src/webmon_app/reporting/report/models.py +++ b/src/webmon_app/reporting/report/models.py @@ -3,4 +3,5 @@ reporting app. The models are defined in the workflow manager and should be installed on the system before running the app. """ + from workflow.database.report.models import * # noqa: F401, F403 diff --git a/src/webmon_app/reporting/report/view_util.py b/src/webmon_app/reporting/report/view_util.py index 0f0d1980..41d1f04c 100644 --- a/src/webmon_app/reporting/report/view_util.py +++ b/src/webmon_app/reporting/report/view_util.py @@ -4,6 +4,7 @@ @author: M. Doucet, Oak Ridge National Laboratory @copyright: 2014 Oak Ridge National Laboratory """ + import logging import json import datetime diff --git a/src/webmon_app/reporting/reporting_app/settings/develop.py b/src/webmon_app/reporting/reporting_app/settings/develop.py index 4ec81782..a6a5b03c 100644 --- a/src/webmon_app/reporting/reporting_app/settings/develop.py +++ b/src/webmon_app/reporting/reporting_app/settings/develop.py @@ -1,4 +1,5 @@ """This is the settings to be used for a local developer build inside of docker""" + from .base import * # noqa import ldap diff --git a/src/webmon_app/reporting/reporting_app/settings/envtest.py b/src/webmon_app/reporting/reporting_app/settings/envtest.py index ba3a9243..4c829093 100644 --- a/src/webmon_app/reporting/reporting_app/settings/envtest.py +++ b/src/webmon_app/reporting/reporting_app/settings/envtest.py @@ -1,4 +1,5 @@ """This is the settings to be used for a remote test deploy""" + from .base import * # noqa # symbols to make flake8 happy diff --git a/src/webmon_app/reporting/reporting_app/settings/prod.py b/src/webmon_app/reporting/reporting_app/settings/prod.py index 4f9d1573..61272614 100644 --- a/src/webmon_app/reporting/reporting_app/settings/prod.py +++ b/src/webmon_app/reporting/reporting_app/settings/prod.py @@ -1,4 +1,5 @@ """This is the settings to be used for a remote production deploy""" + from .base import * # noqa from os import environ diff --git a/src/webmon_app/reporting/reporting_app/settings/unittest.py b/src/webmon_app/reporting/reporting_app/settings/unittest.py index 524f11c9..4d53e45b 100644 --- a/src/webmon_app/reporting/reporting_app/settings/unittest.py +++ b/src/webmon_app/reporting/reporting_app/settings/unittest.py @@ -1,4 +1,5 @@ """This configuration is meant for running the unittests outside of docker""" + from .base import * # noqa from .base import INSTALLED_APPS diff --git a/src/webmon_app/reporting/reporting_app/view_util.py b/src/webmon_app/reporting/reporting_app/view_util.py index 02a70156..1c1d4f05 100644 --- a/src/webmon_app/reporting/reporting_app/view_util.py +++ b/src/webmon_app/reporting/reporting_app/view_util.py @@ -3,6 +3,7 @@ @copyright: 2014 Oak Ridge National Laboratory """ + from django.conf import settings import logging diff --git a/src/webmon_app/reporting/reporting_app/wsgi.py b/src/webmon_app/reporting/reporting_app/wsgi.py index 47766d89..516b69f1 100644 --- a/src/webmon_app/reporting/reporting_app/wsgi.py +++ b/src/webmon_app/reporting/reporting_app/wsgi.py @@ -13,6 +13,7 @@ framework. """ + import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "reporting.reporting_app.settings.develop") diff --git a/src/webmon_app/reporting/users/views.py b/src/webmon_app/reporting/users/views.py index 0efe6d56..5c3399b5 100644 --- a/src/webmon_app/reporting/users/views.py +++ b/src/webmon_app/reporting/users/views.py @@ -1,6 +1,7 @@ """ User management """ + from django.shortcuts import render, redirect from django.urls import reverse from django.contrib.auth import login, logout, authenticate diff --git a/src/workflow_app/workflow/states.py b/src/workflow_app/workflow/states.py index d4deef0f..b90104c2 100644 --- a/src/workflow_app/workflow/states.py +++ b/src/workflow_app/workflow/states.py @@ -4,6 +4,7 @@ To add an action for a specific queue, add a StateAction class with the name of the queue in lower-case, replacing periods with underscores. """ + from .state_utilities import logged_action from .settings import POSTPROCESS_ERROR, CATALOG_DATA_READY from .settings import REDUCTION_DATA_READY, REDUCTION_CATALOG_DATA_READY diff --git a/src/workflow_app/workflow/workflow_process.py b/src/workflow_app/workflow/workflow_process.py index 94c662e4..00d13733 100644 --- a/src/workflow_app/workflow/workflow_process.py +++ b/src/workflow_app/workflow/workflow_process.py @@ -1,6 +1,7 @@ """ Actual process that each data run must go through. """ + import json import logging import datetime diff --git a/tests/simple_producer.py b/tests/simple_producer.py index 0c6dac8e..d5c06a9f 100644 --- a/tests/simple_producer.py +++ b/tests/simple_producer.py @@ -1,6 +1,7 @@ """ Example of a simple non-listening producer """ + import stomp import json import time diff --git a/tests/workflow_command.py b/tests/workflow_command.py index 4fefaa2e..b9850747 100644 --- a/tests/workflow_command.py +++ b/tests/workflow_command.py @@ -2,6 +2,7 @@ ActiveMQ client used to issue commands to the post-processing workflow. NOTE: Only works for runs that are already in the DB """ + from report.models import Instrument, DataRun from workflow.settings import BROKERS, ICAT_USER, ICAT_PASSCODE import argparse From 6e29da75539645f66acc5e64fa400cb395946ab2 Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Mon, 29 Apr 2024 14:16:36 +1000 Subject: [PATCH 3/4] Update checks --- src/dasmon_app/dasmon_listener/amq_consumer.py | 4 ++-- src/webmon_app/reporting/dasmon/view_util.py | 2 +- src/webmon_app/reporting/reduction/view_util.py | 2 +- tests/test_autoreducer_high_memory.py | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dasmon_app/dasmon_listener/amq_consumer.py b/src/dasmon_app/dasmon_listener/amq_consumer.py index 897248d8..4fda3af9 100644 --- a/src/dasmon_app/dasmon_listener/amq_consumer.py +++ b/src/dasmon_app/dasmon_listener/amq_consumer.py @@ -189,10 +189,10 @@ def on_message(self, frame): # elif 'timestamp' in data_dict: # timestamp = data_dict['timestamp'] for key in data_dict: - if key == "monitors" and type(data_dict[key]) == dict: + if key == "monitors" and isinstance(data_dict[key], dict): for item in data_dict[key]: # Protect against old API - if not type(data_dict[key][item]) == dict: + if not isinstance(data_dict[key][item], dict): key_id = self.retrieve_parameter("monitor_count_%s" % str(item)) store_and_cache( instrument, diff --git a/src/webmon_app/reporting/dasmon/view_util.py b/src/webmon_app/reporting/dasmon/view_util.py index 209481c8..57c8a81e 100644 --- a/src/webmon_app/reporting/dasmon/view_util.py +++ b/src/webmon_app/reporting/dasmon/view_util.py @@ -1083,7 +1083,7 @@ def get_signals(instrument_id): if len(latests) == 0: latests = PVStringCache.objects.filter(instrument=instrument_id, name=item.pv_name) latest = latests.latest("update_time") - if type(latest.value) == float: + if isinstance(latest.value, float): value = "%g" % latest.value else: value = "%s" % latest.value diff --git a/src/webmon_app/reporting/reduction/view_util.py b/src/webmon_app/reporting/reduction/view_util.py index 01902e53..d1ed8579 100644 --- a/src/webmon_app/reporting/reduction/view_util.py +++ b/src/webmon_app/reporting/reduction/view_util.py @@ -80,7 +80,7 @@ def send_template_request(instrument_id, template_dict, user="unknown"): """ use_default = False if "use_default" in template_dict: - if type(template_dict["use_default"]) == bool: + if isinstance(template_dict["use_default"], bool): use_default = template_dict["use_default"] else: use_default = template_dict["use_default"].lower() == "true" diff --git a/tests/test_autoreducer_high_memory.py b/tests/test_autoreducer_high_memory.py index f9bc8d13..2c6a322f 100644 --- a/tests/test_autoreducer_high_memory.py +++ b/tests/test_autoreducer_high_memory.py @@ -1,5 +1,6 @@ """This is to test that the reduction tasks go to the correct autoreducer node depending on if it requires high memoery or not""" + import psycopg2 import requests import time From e034651ad985fb2408a992f8486bd90db42f7a77 Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Mon, 29 Apr 2024 14:36:13 +1000 Subject: [PATCH 4/4] Update codecov --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72cfb78a..7237bdbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,9 @@ jobs: export DJANGO_SETTINGS_MODULE='reporting.reporting_app.settings.unittest' python -m pytest --cov --cov-report=xml --cov-report=term src - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Build documentation run: | conda activate webmon