From 9963daebead33548616bcd974d48bccf4214a041 Mon Sep 17 00:00:00 2001 From: Jeff Kala <48843785+jeffkala@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:13:26 -0600 Subject: [PATCH 1/3] fixes backups not executing git functions #736 (#737) * fixes backups not eecuting git functions #736 --- changes/736.added | 1 + changes/736.fixed | 1 + nautobot_golden_config/jobs.py | 40 ++++++++----- .../nornir_plays/config_backup.py | 58 ++++++++----------- .../nornir_plays/config_compliance.py | 52 +++++++---------- .../nornir_plays/config_intended.py | 55 ++++++++---------- 6 files changed, 98 insertions(+), 109 deletions(-) create mode 100755 changes/736.added create mode 100755 changes/736.fixed diff --git a/changes/736.added b/changes/736.added new file mode 100755 index 00000000..f5d7038d --- /dev/null +++ b/changes/736.added @@ -0,0 +1 @@ +Add a boolean job parameter `fail_job_on_task_failure` which will determine whether a single task failure anywhere in the job-result should result in job-result status of failed vs successful. diff --git a/changes/736.fixed b/changes/736.fixed new file mode 100755 index 00000000..3e02530d --- /dev/null +++ b/changes/736.fixed @@ -0,0 +1 @@ +Fixes repo push and commit not executing if a exception was raised on any task inside a job. diff --git a/nautobot_golden_config/jobs.py b/nautobot_golden_config/jobs.py index 9131841b..149c0947 100644 --- a/nautobot_golden_config/jobs.py +++ b/nautobot_golden_config/jobs.py @@ -118,21 +118,28 @@ def gc_repo_wrapper(self, *args, **kwargs): ) current_repos = get_refreshed_repos(job_obj=self, repo_types=gitrepo_types, data=self.qs) # This is where the specific jobs run method runs via this decorator. - func(self, *args, **kwargs) - now = make_aware(datetime.now()) - self.logger.debug( - f"Finished the {self.Meta.name} job execution.", - extra={"grouping": "GC After Run"}, - ) - if current_repos: - for _, repo in current_repos.items(): - if repo["to_commit"]: - self.logger.debug( - f"Pushing {self.Meta.name} results to repo {repo['repo_obj'].base_url}.", - extra={"grouping": "GC Repo Commit and Push"}, - ) - repo["repo_obj"].commit_with_added(f"{self.Meta.name.upper()} JOB {now}") - repo["repo_obj"].push() + try: + func(self, *args, **kwargs) + except Exception as error: # pylint: disable=broad-exception-caught + error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" + # Raise error only if the job kwarg (checkbox) is selected to do so on the job execution form. + if kwargs["fail_job_on_task_failure"]: + raise NornirNautobotException(error_msg) from error + finally: + now = make_aware(datetime.now()) + self.logger.debug( + f"Finished the {self.Meta.name} job execution.", + extra={"grouping": "GC After Run"}, + ) + if current_repos: + for _, repo in current_repos.items(): + if repo["to_commit"]: + self.logger.debug( + f"Pushing {self.Meta.name} results to repo {repo['repo_obj'].base_url}.", + extra={"grouping": "GC Repo Commit and Push"}, + ) + repo["repo_obj"].commit_with_added(f"{self.Meta.name.upper()} JOB {now}") + repo["repo_obj"].push() return gc_repo_wrapper @@ -161,6 +168,9 @@ class FormEntry: # pylint disable=too-few-public-method label="Device Status", ) debug = BooleanVar(description="Enable for more verbose debug logging") + fail_job_on_task_failure = BooleanVar( + description="If any device in the tasks list fails, fail the entire job result." + ) class GoldenConfigJobMixin(Job): # pylint: disable=abstract-method diff --git a/nautobot_golden_config/nornir_plays/config_backup.py b/nautobot_golden_config/nornir_plays/config_backup.py index 69d12b01..95f55e9d 100644 --- a/nautobot_golden_config/nornir_plays/config_backup.py +++ b/nautobot_golden_config/nornir_plays/config_backup.py @@ -10,12 +10,11 @@ from nornir import InitNornir from nornir.core.plugins.inventory import InventoryPluginRegister from nornir.core.task import Result, Task -from nornir_nautobot.exceptions import NornirNautobotException from nornir_nautobot.plugins.tasks.dispatcher import dispatcher from nautobot_golden_config.models import ConfigRemove, ConfigReplace, GoldenConfig from nautobot_golden_config.nornir_plays.processor import ProcessGoldenConfig from nautobot_golden_config.utilities.db_management import close_threaded_db_connections -from nautobot_golden_config.utilities.helper import ( # get_device_to_settings_map,; get_job_filter, +from nautobot_golden_config.utilities.helper import ( dispatch_params, render_jinja_template, verify_settings, @@ -103,36 +102,29 @@ def config_backup(job_result, log_level, qs, device_to_settings_map): if not replace_regex_dict.get(regex.platform.network_driver): replace_regex_dict[regex.platform.network_driver] = [] replace_regex_dict[regex.platform.network_driver].append({"replace": regex.replace, "regex": regex.regex}) - try: - with InitNornir( - runner=NORNIR_SETTINGS.get("runner"), - logging={"enabled": False}, - inventory={ - "plugin": "nautobot-inventory", - "options": { - "credentials_class": NORNIR_SETTINGS.get("credentials"), - "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": qs, - "defaults": {"now": now}, - }, + with InitNornir( + runner=NORNIR_SETTINGS.get("runner"), + logging={"enabled": False}, + inventory={ + "plugin": "nautobot-inventory", + "options": { + "credentials_class": NORNIR_SETTINGS.get("credentials"), + "params": NORNIR_SETTINGS.get("inventory_params"), + "queryset": qs, + "defaults": {"now": now}, }, - ) as nornir_obj: - nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) - - logger.debug("Run nornir backup tasks.") - nr_with_processors.run( - task=run_backup, - name="BACKUP CONFIG", - logger=logger, - device_to_settings_map=device_to_settings_map, - remove_regex_dict=remove_regex_dict, - replace_regex_dict=replace_regex_dict, - ) - logger.debug("Completed configuration from devices.") - - except Exception as error: - error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" - logger.error(error_msg) - raise NornirNautobotException(error_msg) from error - + }, + ) as nornir_obj: + nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) + + logger.debug("Run nornir backup tasks.") + nr_with_processors.run( + task=run_backup, + name="BACKUP CONFIG", + logger=logger, + device_to_settings_map=device_to_settings_map, + remove_regex_dict=remove_regex_dict, + replace_regex_dict=replace_regex_dict, + ) + logger.debug("Completed configuration from devices.") logger.debug("Completed configuration backup job for devices.") diff --git a/nautobot_golden_config/nornir_plays/config_compliance.py b/nautobot_golden_config/nornir_plays/config_compliance.py index af511b4c..296956ea 100644 --- a/nautobot_golden_config/nornir_plays/config_compliance.py +++ b/nautobot_golden_config/nornir_plays/config_compliance.py @@ -177,35 +177,27 @@ def config_compliance(job_result, log_level, qs, device_to_settings_map): for settings in set(device_to_settings_map.values()): verify_settings(logger, settings, ["backup_path_template", "intended_path_template"]) - - try: - with InitNornir( - runner=NORNIR_SETTINGS.get("runner"), - logging={"enabled": False}, - inventory={ - "plugin": "nautobot-inventory", - "options": { - "credentials_class": NORNIR_SETTINGS.get("credentials"), - "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": qs, - "defaults": {"now": now}, - }, + with InitNornir( + runner=NORNIR_SETTINGS.get("runner"), + logging={"enabled": False}, + inventory={ + "plugin": "nautobot-inventory", + "options": { + "credentials_class": NORNIR_SETTINGS.get("credentials"), + "params": NORNIR_SETTINGS.get("inventory_params"), + "queryset": qs, + "defaults": {"now": now}, }, - ) as nornir_obj: - nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) - - logger.debug("Run nornir compliance tasks.") - nr_with_processors.run( - task=run_compliance, - name="RENDER COMPLIANCE TASK GROUP", - logger=logger, - device_to_settings_map=device_to_settings_map, - rules=rules, - ) - - except Exception as error: - error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" - logger.error(error_msg) - raise NornirNautobotException(error_msg) from error - + }, + ) as nornir_obj: + nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) + + logger.debug("Run nornir compliance tasks.") + nr_with_processors.run( + task=run_compliance, + name="RENDER COMPLIANCE TASK GROUP", + logger=logger, + device_to_settings_map=device_to_settings_map, + rules=rules, + ) logger.debug("Completed compliance job for devices.") diff --git a/nautobot_golden_config/nornir_plays/config_intended.py b/nautobot_golden_config/nornir_plays/config_intended.py index 0b24b7b5..fbfc8104 100644 --- a/nautobot_golden_config/nornir_plays/config_intended.py +++ b/nautobot_golden_config/nornir_plays/config_intended.py @@ -110,35 +110,28 @@ def config_intended(job_result, log_level, job_class_instance, qs, device_to_set # Retrieve filters from the Django jinja template engine jinja_env = get_django_env() - - try: - with InitNornir( - runner=NORNIR_SETTINGS.get("runner"), - logging={"enabled": False}, - inventory={ - "plugin": "nautobot-inventory", - "options": { - "credentials_class": NORNIR_SETTINGS.get("credentials"), - "params": NORNIR_SETTINGS.get("inventory_params"), - "queryset": qs, - "defaults": {"now": now}, - }, + with InitNornir( + runner=NORNIR_SETTINGS.get("runner"), + logging={"enabled": False}, + inventory={ + "plugin": "nautobot-inventory", + "options": { + "credentials_class": NORNIR_SETTINGS.get("credentials"), + "params": NORNIR_SETTINGS.get("inventory_params"), + "queryset": qs, + "defaults": {"now": now}, }, - ) as nornir_obj: - nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) - - logger.debug("Run nornir render config tasks.") - # Run the Nornir Tasks - nr_with_processors.run( - task=run_template, - name="RENDER CONFIG", - logger=logger, - device_to_settings_map=device_to_settings_map, - job_class_instance=job_class_instance, - jinja_env=jinja_env, - ) - - except Exception as error: - error_msg = f"`E3001:` General Exception handler, original error message ```{error}```" - logger.error(error_msg) - raise NornirNautobotException(error_msg) from error + }, + ) as nornir_obj: + nr_with_processors = nornir_obj.with_processors([ProcessGoldenConfig(logger)]) + + logger.debug("Run nornir render config tasks.") + # Run the Nornir Tasks + nr_with_processors.run( + task=run_template, + name="RENDER CONFIG", + logger=logger, + device_to_settings_map=device_to_settings_map, + job_class_instance=job_class_instance, + jinja_env=jinja_env, + ) From f7ca5f3ce5e24512e91fe60663ec5fdd5ebcaea0 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 14 Mar 2024 12:14:14 -0500 Subject: [PATCH 2/3] prepping 2.0.3 release --- development/towncrier_template.j2 | 2 +- docs/admin/release_notes/version_2.0.md | 11 +++++++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/development/towncrier_template.j2 b/development/towncrier_template.j2 index b0f6e12f..9236d24a 100644 --- a/development/towncrier_template.j2 +++ b/development/towncrier_template.j2 @@ -1,6 +1,6 @@ {% if render_title %} -## [v{{ versiondata.version }} ({{ versiondata.date }})]({{ cookiecutter.repo_url }}/releases/tag/v{{ versiondata.version}}) +## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/nautobot/nautobot-app-golden-config/releases/tag/v{{ versiondata.version}}) {% endif %} {% for section, _ in sections.items() %} diff --git a/docs/admin/release_notes/version_2.0.md b/docs/admin/release_notes/version_2.0.md index 19455831..5b9ca173 100755 --- a/docs/admin/release_notes/version_2.0.md +++ b/docs/admin/release_notes/version_2.0.md @@ -11,6 +11,17 @@ !!! note Please see [migrating guide](../migrating_to_v2.md) for details on migration. + +## v2.0.3 2024-03 + +### Added + +- [#736](https://github.com/nautobot/nautobot-app-golden-config/issues/736) - Add a boolean job parameter `fail_job_on_task_failure` which will determine whether a single task failure anywhere in the job-result should result in job-result status of failed vs successful. + +### Fixed + +- [#736](https://github.com/nautobot/nautobot-app-golden-config/issues/736) - Fixes repo push and commit not executing if a exception was raised on any task inside a job. + ## v2.0.2 - 2024-03 ### Added diff --git a/pyproject.toml b/pyproject.toml index 6581c770..9ed66c3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nautobot-golden-config" -version = "2.0.2" +version = "2.0.3" description = "An app for configuration on nautobot" authors = ["Network to Code, LLC "] license = "Apache-2.0" From f0b4c7398913cc9702fe1efa181a5de5a3311455 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 14 Mar 2024 15:37:31 -0500 Subject: [PATCH 3/3] add change fragment --- changes/739.added | 1 + 1 file changed, 1 insertion(+) create mode 100755 changes/739.added diff --git a/changes/739.added b/changes/739.added new file mode 100755 index 00000000..ac17d983 --- /dev/null +++ b/changes/739.added @@ -0,0 +1 @@ +Prep release v2.0.3