diff --git a/.gitignore b/.gitignore index 2d3d689..487dd3e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ uv.lock # docs /docs/generated/ /docs/_build/ + +# Windows +Thumbs.db +~$* diff --git a/CHANGELOG.md b/CHANGELOG.md index cc630e0..09c3e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,34 @@ and this project adheres to [Semantic Versioning][]. [keep a changelog]: https://keepachangelog.com/en/1.0.0/ [semantic versioning]: https://semver.org/spec/v2.0.0.html -## [Unreleased] +## v0.9.0 ### New Features - `dso watermark` now supports files in PDF format. With this change, quarto reports using the watermark feature can - be rendered to PDF, too. + be rendered to PDF, too ([#26](https://github.com/Boehringer-Ingelheim/dso/pull/26)). + +### Fixes + +- Fix linting rule DSO001: It is now allowed to specify additional arguments in `read_params()`, e.g. `quiet = TRUE` ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36)). +- It is now possible to use Jinja2 interpolation in combination with `!path` objects ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36)) +- Improve error messages when `dso get-config` can't find required input files ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36)) + +### Documentation + +- Documentation is now built via sphinx and hosted on GitHub pages: https://boehringer-ingelheim.github.io/dso/ ([#35](https://github.com/Boehringer-Ingelheim/dso/pull/35)). + +### Template updates + +- Make instruction comments in quarto template more descriptive ([#33](https://github.com/Boehringer-Ingelheim/dso/pull/33)). +- Include `params.yaml` in default project `.gitignore`. We decided to not track `params.yaml` in git anymore + since it adds noise during code review and led to merge conflicts in some cases. In the future, a certain + `dso` version will be tied to each project, improving reproducibility also without `params.yaml` files. + +### Migration advice + +- Add `params.yaml` to your project-level `.gitignore`. Then execute `find -iname "params.yaml" -exec git rm --cached {} \;` + to untrack existing `params.yaml` files. ## v0.8.2 @@ -53,7 +75,6 @@ and this project adheres to [Semantic Versioning][]. - When running `dso repro`, configuration is only compiled once and not recompiled when `dso exec` or `dso get-config` is called internally. This reduces runtime and redundant log messages. - ## v0.7.0 - Improved watermarking support diff --git a/pyproject.toml b/pyproject.toml index fd779a1..8af13b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,16 +28,14 @@ dependencies = [ "jinja2", "panflute", "pillow", - "platformdirs", "pre-commit", "pypdf", "pyyaml", "questionary", + "rich", "rich-click", - # "hiyapyco", # using vendored code now "ruamel-yaml", "svgutils", - "tqdm", ] optional-dependencies.dev = [ "hatch", "pre-commit" ] @@ -88,6 +86,8 @@ scripts.clean = "git clean -fdX -- {args:docs}" [tool.hatch.envs.hatch-test] features = [ "test" ] +[[tool.hatch.envs.hatch-test.matrix]] +python = [ "3.12" ] [tool.ruff] line-length = 120 diff --git a/src/dso/_logging.py b/src/dso/_logging.py index 21f41ec..8510386 100644 --- a/src/dso/_logging.py +++ b/src/dso/_logging.py @@ -2,7 +2,6 @@ from rich.console import Console from rich.logging import RichHandler -from rich.traceback import install console_stderr = Console(stderr=True) console = Console(stderr=False) @@ -12,4 +11,3 @@ format="%(message)s", handlers=[RichHandler(markup=True, console=console_stderr, show_path=False, show_time=True)], ) -install(show_locals=True) diff --git a/src/dso/compile_config.py b/src/dso/compile_config.py index 16e8a2d..cec0628 100644 --- a/src/dso/compile_config.py +++ b/src/dso/compile_config.py @@ -52,8 +52,16 @@ def _load_yaml_with_auto_adjusting_paths(yaml_stream: TextIOWrapper, destination if not destination.is_relative_to(source): raise ValueError("Destination path can be the same as source, or a child thereof.") + # inherit from `str` to make this compatible with hiyapyco interpolation @yaml_object(ruamel) - class AutoAdjustingPathWithLocation: + class AutoAdjustingPathWithLocation(str): + """ + Represents a YAML node that adjusts a relative path relative to a specified destination directory. + + Can be evaulated either using Ruamel during dumping YAML to file, or whenever it is cast + to a string (e.g. by hiyapyco). To this end, __repr__ and __str__ are overridden. + """ + yaml_tag = "!path" def __init__(self, path: str): @@ -71,6 +79,12 @@ def get_adjusted(self): def to_yaml(cls, representer, node): return representer.represent_str(str(node.get_adjusted())) + def __repr__(self): + return str(self.get_adjusted()) + + def __str__(self): + return str(self.get_adjusted()) + @classmethod def from_yaml(cls, constructor, node): return cls(node.value) diff --git a/src/dso/get_config.py b/src/dso/get_config.py index 1c31109..924ed1c 100644 --- a/src/dso/get_config.py +++ b/src/dso/get_config.py @@ -63,15 +63,29 @@ def get_config(stage: str, *, all: bool = False, skip_compile: bool = False) -> log.debug("Skipping compilation of configuration") compile_all_configs([stage_path]) yaml = YAML(typ="safe") - config = yaml.load(stage_path / "params.yaml") + + try: + config = yaml.load(stage_path / "params.yaml") + except OSError: + log.error("No params.yaml (or compilable params.in.yaml) found in directory.") + sys.exit(1) if all: return config else: - dvc_config = yaml.load(stage_path / "dvc.yaml") - dvc_stages = dvc_config.get("stages", None) + try: + dvc_config = yaml.load(stage_path / "dvc.yaml") + except OSError: + log.error("No dvc.yaml found in directory.") + sys.exit(1) + + try: + dvc_stages = dvc_config.get("stages", None) + except AttributeError: + dvc_stages = None + if not dvc_stages: - log.error("At least one stage must be defined in `dvc.yaml`") + log.error("At least one stage must be defined in `dvc.yaml` (unless --all is specified)") sys.exit(1) elif len(dvc_stages) > 1 and stage_name is None: log.error( diff --git a/src/dso/lint.py b/src/dso/lint.py index 1b3d001..304e211 100644 --- a/src/dso/lint.py +++ b/src/dso/lint.py @@ -87,13 +87,13 @@ def check(cls, file): # .parent to remove the dvc.yaml filename stage_path_expected = str(stage_path_expected.parent.relative_to(root_path)) content = file.read_text() - pattern = r"params\s*(=|<-)\s*(dso::)?read_params\s*\(([\s\S]*?)\)" + pattern = r"[\s\S]*?(dso::)?read_params\s*\(([\s\S]*?)(\s*,.*)?\)" res = re.findall(pattern, content, flags=re.MULTILINE) if len(res) == 0: raise LintError(f"no `params = read_params('{stage_path_expected}')` statement found in qmd document") if len(res) > 1: raise LintError("Multiple read_params statements found") - stage_path = res[0][2].strip().strip("'\"").rstrip("/") # get what's within the brackets for read_params + stage_path = res[0][1].strip().strip("'\"").rstrip("/") # get what's within the brackets for read_params if stage_path_expected != stage_path: raise LintError( f"Stage path specified in read_params doesn't match. Expected: {stage_path_expected}, Actual: {stage_path}" diff --git a/src/dso/templates/init/default/.gitignore b/src/dso/templates/init/default/.gitignore index eb33794..9757021 100644 --- a/src/dso/templates/init/default/.gitignore +++ b/src/dso/templates/init/default/.gitignore @@ -1,3 +1,7 @@ +# dso +.dso.jso +params.yaml + # Editors *.code-workspace .vscode @@ -37,5 +41,6 @@ sccprj/ # nodejs/pre-commit /node_modules -# dso -.dso.json +# Windows +Thumbs.db +~$* diff --git a/tests/test_compile_config.py b/tests/test_compile_config.py index df2c5d2..fb611c3 100644 --- a/tests/test_compile_config.py +++ b/tests/test_compile_config.py @@ -1,3 +1,4 @@ +from functools import partial from io import StringIO from pathlib import Path from textwrap import dedent @@ -7,6 +8,7 @@ from click.testing import CliRunner from ruamel.yaml import YAML +from dso import hiyapyco from dso.compile_config import ( _get_list_of_configs_to_compile, _get_parent_configs, @@ -23,7 +25,14 @@ def _setup_yaml_configs(tmp_path, configs: dict[str, dict]): yaml.dump(dict, f) -def test_auto_adjusting_path(tmp_path): +@pytest.mark.parametrize("interpolate", [True, False]) +def test_auto_adjusting_path(tmp_path, interpolate): + """Test that audo-adjusting paths work as expected. + + If `interpolate` is `True`, the AutoAdjustingPath object + is already evaluated by hiyapyco, otherwise it is returned + as an object that can be dumped by ruamel using the custom representer. + """ test_file = tmp_path / "params.in.yaml" destination = tmp_path / "subproject1" / "stageA" destination.mkdir(parents=True) @@ -38,14 +47,56 @@ def test_auto_adjusting_path(tmp_path): ) ) with test_file.open("r") as f: - res = list(_load_yaml_with_auto_adjusting_paths(f, destination)) + res = hiyapyco.load( + str(test_file), + method=hiyapyco.METHOD_MERGE, + interpolate=interpolate, + loader_callback=partial(_load_yaml_with_auto_adjusting_paths, destination=destination), + ) ruamel = YAML() with StringIO() as s: ruamel.dump(res, s) actual = s.getvalue() - assert actual.strip() == "- my_path: ../../test.txt" + assert actual.strip() == "my_path: ../../test.txt" + + +@pytest.mark.parametrize( + "test_yaml,expected", + [ + ( + """\ + A: !path dir_A + B: "{{ A }}/B.txt" + """, + "dir_A/B.txt", + ), + ( + """\ + A: dir_A + B: !path "{{ A }}/B.txt" + """, + "dir_A/B.txt", + ), + ], +) +def test_auto_adjusting_path_with_jinja(tmp_path, test_yaml, expected): + runner = CliRunner() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + td = Path(td) + test_file = td / "params.in.yaml" + (td / ".git").mkdir() + + with test_file.open("w") as f: + f.write(dedent(test_yaml)) + + result = runner.invoke(cli, []) + print(result.output) + td = Path(td) + assert result.exit_code == 0 + with (td / "params.yaml").open() as f: + assert yaml.safe_load(f)["B"] == expected def test_compile_configs(tmp_path): diff --git a/tests/test_lint.py b/tests/test_lint.py index 80aa435..bb3ea91 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -77,9 +77,21 @@ class MockQuartoRule(QuartoRule): """params = read_params("quarto_stage")""", None, ), + ( + """params = read_params("quarto_stage", quiet=TRUE)""", + None, + ), + ( + """params = read_params("quarto_stage"\n, quiet=TRUE)""", + None, + ), ( """foo = read_params("quarto_stage")""", - LintError, + None, + ), + ( + """read_params("quarto_stage")""", + None, ), ( """\ diff --git a/tests/test_util.py b/tests/test_util.py index ca5a0fb..20ffb20 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -48,7 +48,6 @@ def test_git_list_files(dso_project): assert files == [ dso_project / x for x in [ - "params.yaml", ".dvc/.gitignore", ".dvc/config", ".dvcignore",