Release v0.9 (#36)

* Update changelog

* Include params.yaml in project-level gitignore

* Fix linting

* Update linting rules

* Fix DSO001 linting tests

* Fix issue with templating in audoadjusting paths

* Update CHANGELOG

* Fix auto-adjusting paths

* Improve error message for compile-config

* Update changelog

.gitignore

@@ -21,3 +21,7 @@ uv.lock
 # docs
 /docs/generated/
 /docs/_build/
+
+# Windows
+Thumbs.db
+~$*

CHANGELOG.md

@@ -8,12 +8,34 @@ and this project adheres to [Semantic Versioning][].
 [keep a changelog]: https://keepachangelog.com/en/1.0.0/
 [semantic versioning]: https://semver.org/spec/v2.0.0.html
 
-## [Unreleased]
+## v0.9.0
 
 ### New Features
 
 -   `dso watermark` now supports files in PDF format. With this change, quarto reports using the watermark feature can
-    be rendered to PDF, too.
+    be rendered to PDF, too ([#26](https://github.com/Boehringer-Ingelheim/dso/pull/26)).
+
+### Fixes
+
+-   Fix linting rule DSO001: It is now allowed to specify additional arguments in `read_params()`, e.g. `quiet = TRUE` ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36)).
+-   It is now possible to use Jinja2 interpolation in combination with `!path` objects ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36))
+-   Improve error messages when `dso get-config` can't find required input files ([#36](https://github.com/Boehringer-Ingelheim/dso/pull/36))
+
+### Documentation
+
+-   Documentation is now built via sphinx and hosted on GitHub pages: https://boehringer-ingelheim.github.io/dso/ ([#35](https://github.com/Boehringer-Ingelheim/dso/pull/35)).
+
+### Template updates
+
+-   Make instruction comments in quarto template more descriptive ([#33](https://github.com/Boehringer-Ingelheim/dso/pull/33)).
+-   Include `params.yaml` in default project `.gitignore`. We decided to not track `params.yaml` in git anymore
+    since it adds noise during code review and led to merge conflicts in some cases. In the future, a certain
+    `dso` version will be tied to each project, improving reproducibility also without `params.yaml` files.
+
+### Migration advice
+
+-   Add `params.yaml` to your project-level `.gitignore`. Then execute `find -iname "params.yaml" -exec git rm --cached {} \;`
+    to untrack existing `params.yaml` files.
 
 ## v0.8.2
 
@@ -53,7 +75,6 @@ and this project adheres to [Semantic Versioning][].
 -   When running `dso repro`, configuration is only compiled once and not recompiled when `dso exec` or `dso get-config`
     is called internally. This reduces runtime and redundant log messages.
 
-
 ## v0.7.0
 
 -   Improved watermarking support

pyproject.toml

@@ -28,16 +28,14 @@ dependencies = [
   "jinja2",
   "panflute",
   "pillow",
-  "platformdirs",
   "pre-commit",
   "pypdf",
   "pyyaml",
   "questionary",
+  "rich",
   "rich-click",
-  #    "hiyapyco",  # using vendored code now
   "ruamel-yaml",
   "svgutils",
-  "tqdm",
 ]
 
 optional-dependencies.dev = [ "hatch", "pre-commit" ]
@@ -88,6 +86,8 @@ scripts.clean = "git clean -fdX -- {args:docs}"
 
 [tool.hatch.envs.hatch-test]
 features = [ "test" ]
+[[tool.hatch.envs.hatch-test.matrix]]
+python = [ "3.12" ]
 
 [tool.ruff]
 line-length = 120

src/dso/_logging.py

@@ -2,7 +2,6 @@
 
 from rich.console import Console
 from rich.logging import RichHandler
-from rich.traceback import install
 
 console_stderr = Console(stderr=True)
 console = Console(stderr=False)
@@ -12,4 +11,3 @@
     format="%(message)s",
     handlers=[RichHandler(markup=True, console=console_stderr, show_path=False, show_time=True)],
 )
-install(show_locals=True)

src/dso/compile_config.py

@@ -52,8 +52,16 @@ def _load_yaml_with_auto_adjusting_paths(yaml_stream: TextIOWrapper, destination
     if not destination.is_relative_to(source):
         raise ValueError("Destination path can be the same as source, or a child thereof.")
 
+    # inherit from `str` to make this compatible with hiyapyco interpolation
     @yaml_object(ruamel)
-    class AutoAdjustingPathWithLocation:
+    class AutoAdjustingPathWithLocation(str):
+        """
+        Represents a YAML node that adjusts a relative path relative to a specified destination directory.
+
+        Can be evaulated either using Ruamel during dumping YAML to file, or whenever it is cast
+        to a string (e.g. by hiyapyco). To this end, __repr__ and __str__ are overridden.
+        """
+
         yaml_tag = "!path"
 
         def __init__(self, path: str):
@@ -71,6 +79,12 @@ def get_adjusted(self):
         def to_yaml(cls, representer, node):
             return representer.represent_str(str(node.get_adjusted()))
 
+        def __repr__(self):
+            return str(self.get_adjusted())
+
+        def __str__(self):
+            return str(self.get_adjusted())
+
         @classmethod
         def from_yaml(cls, constructor, node):
             return cls(node.value)

src/dso/get_config.py

@@ -63,15 +63,29 @@ def get_config(stage: str, *, all: bool = False, skip_compile: bool = False) ->
         log.debug("Skipping compilation of configuration")
         compile_all_configs([stage_path])
     yaml = YAML(typ="safe")
-    config = yaml.load(stage_path / "params.yaml")
+
+    try:
+        config = yaml.load(stage_path / "params.yaml")
+    except OSError:
+        log.error("No params.yaml (or compilable params.in.yaml) found in directory.")
+        sys.exit(1)
 
     if all:
         return config
     else:
-        dvc_config = yaml.load(stage_path / "dvc.yaml")
-        dvc_stages = dvc_config.get("stages", None)
+        try:
+            dvc_config = yaml.load(stage_path / "dvc.yaml")
+        except OSError:
+            log.error("No dvc.yaml found in directory.")
+            sys.exit(1)
+
+        try:
+            dvc_stages = dvc_config.get("stages", None)
+        except AttributeError:
+            dvc_stages = None
+
         if not dvc_stages:
-            log.error("At least one stage must be defined in `dvc.yaml`")
+            log.error("At least one stage must be defined in `dvc.yaml` (unless --all is specified)")
             sys.exit(1)
         elif len(dvc_stages) > 1 and stage_name is None:
             log.error(

src/dso/lint.py

@@ -87,13 +87,13 @@ def check(cls, file):
         # .parent to remove the dvc.yaml filename
         stage_path_expected = str(stage_path_expected.parent.relative_to(root_path))
         content = file.read_text()
-        pattern = r"params\s*(=|<-)\s*(dso::)?read_params\s*\(([\s\S]*?)\)"
+        pattern = r"[\s\S]*?(dso::)?read_params\s*\(([\s\S]*?)(\s*,.*)?\)"
         res = re.findall(pattern, content, flags=re.MULTILINE)
         if len(res) == 0:
             raise LintError(f"no `params = read_params('{stage_path_expected}')` statement found in qmd document")
         if len(res) > 1:
             raise LintError("Multiple read_params statements found")
-        stage_path = res[0][2].strip().strip("'\"").rstrip("/")  # get what's within the brackets for read_params
+        stage_path = res[0][1].strip().strip("'\"").rstrip("/")  # get what's within the brackets for read_params
         if stage_path_expected != stage_path:
             raise LintError(
                 f"Stage path specified in read_params doesn't match. Expected: {stage_path_expected}, Actual: {stage_path}"

src/dso/templates/init/default/.gitignore

@@ -1,3 +1,7 @@
+# dso
+.dso.jso
+params.yaml
+
 # Editors
 *.code-workspace
 .vscode
@@ -37,5 +41,6 @@ sccprj/
 # nodejs/pre-commit
 /node_modules
 
-# dso
-.dso.json
+# Windows
+Thumbs.db
+~$*

tests/test_compile_config.py

@@ -1,3 +1,4 @@
+from functools import partial
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
@@ -7,6 +8,7 @@
 from click.testing import CliRunner
 from ruamel.yaml import YAML
 
+from dso import hiyapyco
 from dso.compile_config import (
     _get_list_of_configs_to_compile,
     _get_parent_configs,
@@ -23,7 +25,14 @@ def _setup_yaml_configs(tmp_path, configs: dict[str, dict]):
             yaml.dump(dict, f)
 
 
-def test_auto_adjusting_path(tmp_path):
+@pytest.mark.parametrize("interpolate", [True, False])
+def test_auto_adjusting_path(tmp_path, interpolate):
+    """Test that audo-adjusting paths work as expected.
+
+    If `interpolate` is `True`, the AutoAdjustingPath object
+    is already evaluated by hiyapyco, otherwise it is returned
+    as an object that can be dumped by ruamel using the custom representer.
+    """
     test_file = tmp_path / "params.in.yaml"
     destination = tmp_path / "subproject1" / "stageA"
     destination.mkdir(parents=True)
@@ -38,14 +47,56 @@ def test_auto_adjusting_path(tmp_path):
             )
         )
     with test_file.open("r") as f:
-        res = list(_load_yaml_with_auto_adjusting_paths(f, destination))
+        res = hiyapyco.load(
+            str(test_file),
+            method=hiyapyco.METHOD_MERGE,
+            interpolate=interpolate,
+            loader_callback=partial(_load_yaml_with_auto_adjusting_paths, destination=destination),
+        )
 
     ruamel = YAML()
     with StringIO() as s:
         ruamel.dump(res, s)
         actual = s.getvalue()
 
-    assert actual.strip() == "- my_path: ../../test.txt"
+    assert actual.strip() == "my_path: ../../test.txt"
+
+
+@pytest.mark.parametrize(
+    "test_yaml,expected",
+    [
+        (
+            """\
+            A: !path dir_A
+            B: "{{ A }}/B.txt"
+            """,
+            "dir_A/B.txt",
+        ),
+        (
+            """\
+            A: dir_A
+            B: !path "{{ A }}/B.txt"
+            """,
+            "dir_A/B.txt",
+        ),
+    ],
+)
+def test_auto_adjusting_path_with_jinja(tmp_path, test_yaml, expected):
+    runner = CliRunner()
+    with runner.isolated_filesystem(temp_dir=tmp_path) as td:
+        td = Path(td)
+        test_file = td / "params.in.yaml"
+        (td / ".git").mkdir()
+
+        with test_file.open("w") as f:
+            f.write(dedent(test_yaml))
+
+        result = runner.invoke(cli, [])
+        print(result.output)
+        td = Path(td)
+        assert result.exit_code == 0
+        with (td / "params.yaml").open() as f:
+            assert yaml.safe_load(f)["B"] == expected
 
 
 def test_compile_configs(tmp_path):

tests/test_lint.py

@@ -77,9 +77,21 @@ class MockQuartoRule(QuartoRule):
             """params = read_params("quarto_stage")""",
             None,
         ),
+        (
+            """params = read_params("quarto_stage", quiet=TRUE)""",
+            None,
+        ),
+        (
+            """params = read_params("quarto_stage"\n, quiet=TRUE)""",
+            None,
+        ),
         (
             """foo = read_params("quarto_stage")""",
-            LintError,
+            None,
+        ),
+        (
+            """read_params("quarto_stage")""",
+            None,
         ),
         (
             """\

tests/test_util.py

@@ -48,7 +48,6 @@ def test_git_list_files(dso_project):
     assert files == [
         dso_project / x
         for x in [
-            "params.yaml",
             ".dvc/.gitignore",
             ".dvc/config",
             ".dvcignore",