From b8dedd9ad78548a3e4b6992ef164c0775688c898 Mon Sep 17 00:00:00 2001 From: Simon de Vlieger Date: Fri, 12 Apr 2024 22:23:32 +0200 Subject: [PATCH] poc: initial implementation This is the initial implementation of the Proof of Concept for the Omnifest Toolkit (`otk`). `otk` is implemented in Python as it's the shared language of choice between the creators and the users of `osbuild-mpp` which `otk` aims to improve upon. The `otk` binary is laid out as follows: - `otk compile` compiles an omnifest into its targets. The `otk` package is laid out as follows: - `otk/parse` contains parsing code for full schemas and single primitives such as names or variables, it also contains wrapping types for internal types. - `otk/help` contains helper functions. - `otk/transform` contains the logic for transforming trees including the directive implementations in the directive submodule. - `otk/error.py` contains `otk` error types, all errors inherit from the `OTKError` type. - `otk/tree.py` contains types and functions to deal with the tree transformations that are at the core of `otk`. - `otk/command.py` contains the subcommands as supported by the `otk` entry point. Other than the above the repository also contains: - `test/` contains unittests for `otk`. - `doc/` contains Sphinx documentation for `otk`. - `example/` contains YAML example files for `otk`. Signed-off-by: Simon de Vlieger --- .gitignore | 5 + Makefile | 13 +- README.md | 2 +- doc/directives.md | 11 +- example/fedora/fedora-39-amd64.yaml | 6 - example/fedora/minimal-40-x86_64.yaml | 71 ++++++++ example/fedora/minimal.yaml | 59 ------ example/fedora/osbuild/pipeline/image.yaml | 2 + example/fedora/osbuild/pipeline/raw.yaml | 17 ++ example/fedora/osbuild/pipeline/tree.yaml | 68 +++++++ example/fedora/osbuild/pipeline/xz.yaml | 2 + example/fedora/osbuild/pipelines/root.yaml | 17 -- example/fedora/osbuild/pipelines/tree.yaml | 66 ------- example/fedora/osbuild/root.yaml | 28 +++ example/fedora/repositories/38.yaml | 6 - example/fedora/repositories/39.yaml | 6 - example/fedora/repositories/40.yaml | 6 - example/fedora/repositories/rawhide.yaml | 6 - .../raw.yaml => repository/40-gpg.yaml} | 0 example/fedora/repository/40.yaml | 2 + pyproject.toml | 27 +++ src/otk/__init__.py | 0 src/otk/__main__.py | 3 + src/otk/command.py | 108 +++++++++++ src/otk/error.py | 88 +++++++++ src/otk/help/__init__.py | 0 src/otk/help/log.py | 37 ++++ src/otk/parse/__init__.py | 0 src/otk/parse/document.py | 74 ++++++++ src/otk/parse/schema/__init__.py | 0 src/otk/parse/schema/omnifest.py | 0 src/otk/parse/token/__init__.py | 0 src/otk/parse/token/name.py | 0 src/otk/parse/token/variable.py | 0 src/otk/transform/__init__.py | 124 +++++++++++++ src/otk/transform/context.py | 76 ++++++++ src/otk/transform/directive.py | 170 ++++++++++++++++++ src/otk/tree.py | 57 ++++++ test/test_parse.py | 0 test/test_parse_document.py | 66 +++++++ test/test_transform_context.py | 65 +++++++ test/test_transform_directive.py | 111 ++++++++++++ test/test_tree.py | 0 43 files changed, 1221 insertions(+), 178 deletions(-) create mode 100644 .gitignore delete mode 100644 example/fedora/fedora-39-amd64.yaml create mode 100644 example/fedora/minimal-40-x86_64.yaml delete mode 100644 example/fedora/minimal.yaml create mode 100644 example/fedora/osbuild/pipeline/image.yaml create mode 100644 example/fedora/osbuild/pipeline/raw.yaml create mode 100644 example/fedora/osbuild/pipeline/tree.yaml create mode 100644 example/fedora/osbuild/pipeline/xz.yaml delete mode 100644 example/fedora/osbuild/pipelines/root.yaml delete mode 100644 example/fedora/osbuild/pipelines/tree.yaml create mode 100644 example/fedora/osbuild/root.yaml delete mode 100644 example/fedora/repositories/38.yaml delete mode 100644 example/fedora/repositories/39.yaml delete mode 100644 example/fedora/repositories/40.yaml delete mode 100644 example/fedora/repositories/rawhide.yaml rename example/fedora/{osbuild/pipelines/raw.yaml => repository/40-gpg.yaml} (100%) create mode 100644 example/fedora/repository/40.yaml create mode 100644 pyproject.toml create mode 100644 src/otk/__init__.py create mode 100644 src/otk/__main__.py create mode 100644 src/otk/command.py create mode 100644 src/otk/error.py create mode 100644 src/otk/help/__init__.py create mode 100644 src/otk/help/log.py create mode 100644 src/otk/parse/__init__.py create mode 100644 src/otk/parse/document.py create mode 100644 src/otk/parse/schema/__init__.py create mode 100644 src/otk/parse/schema/omnifest.py create mode 100644 src/otk/parse/token/__init__.py create mode 100644 src/otk/parse/token/name.py create mode 100644 src/otk/parse/token/variable.py create mode 100644 src/otk/transform/__init__.py create mode 100644 src/otk/transform/context.py create mode 100644 src/otk/transform/directive.py create mode 100644 src/otk/tree.py create mode 100644 test/test_parse.py create mode 100644 test/test_parse_document.py create mode 100644 test/test_transform_context.py create mode 100644 test/test_transform_directive.py create mode 100644 test/test_tree.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..376ddde6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv +*.egg-info +__pycache__ +.coverage +htmlcov diff --git a/Makefile b/Makefile index 75cc126d..fed4d36e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,15 @@ +.PHONY: lint lint: @find . -name '*.yaml' | xargs yamllint -.PHONY: lint +.PHONY: type +type: + @mypy src/ test/ + +.PHONY: format +format: + @ruff format src/ test/ + +.PHONY: test +test: + @pytest diff --git a/README.md b/README.md index b211118f..5e0416bd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ There is documentation about the [format](./doc/format.md) and the available ## Examples -Read the [examples](./example). +Read the [examples](./example) to see what omnifests look like. ## Problem(s) diff --git a/doc/directives.md b/doc/directives.md index 2ac05a32..8e186012 100644 --- a/doc/directives.md +++ b/doc/directives.md @@ -12,9 +12,10 @@ other parts of the omnifest. Variable scope is global, an `otk.define` directive anywhere in the omnifest tree will result in the defined names being hoisted to the global scope. -Double definitions of variables are forbidden and will cause an error when -detected. It is thus wise to 'namespace' variables by putting them inside an -map. +Redefinitions of variables are forbidden and will cause an error when detected. +A redefinition of a variable is assigning a value different from the value it +is currently holding. It is thus wise to 'namespace' variables by putting them +inside a map. Expects a `map` for its value. @@ -117,6 +118,8 @@ otk.define: values: - ${a} - ${b} + - - 5 + - 6 ``` ### otk.op.map.join @@ -134,7 +137,7 @@ otk.define: b: b: 2 c: - otk.op.hash.merge: + otk.op.map.merge: values: - ${a} - ${b} diff --git a/example/fedora/fedora-39-amd64.yaml b/example/fedora/fedora-39-amd64.yaml deleted file mode 100644 index 692e6b5d..00000000 --- a/example/fedora/fedora-39-amd64.yaml +++ /dev/null @@ -1,6 +0,0 @@ -otk.define: - version: 39 - architecture: amd64 - -otk.include: - path: minimal.yaml diff --git a/example/fedora/minimal-40-x86_64.yaml b/example/fedora/minimal-40-x86_64.yaml new file mode 100644 index 00000000..a25df04a --- /dev/null +++ b/example/fedora/minimal-40-x86_64.yaml @@ -0,0 +1,71 @@ +otk.version: 1 + +otk.define: + version: 40 + architecture: "aarch64" + packages: + # These packages are used in the buildroot + root: + docs: false + weak: false + packages: + include: + - "@core" + # TODO We can't merge nonexistent vars + exclude: + - "nonexistent" + # These packages are used for the operating system tree which is what ends + # up in the outputs. + tree: + docs: false + weak: false + packages: + include: + - "@core" + - "initial-setup" + - "libxkbcommon" + - "NetworkManager-wifi" + - "brcmfmac-firmware" + - "realtek-firmware" + - "iwlwifi-mvm-firmware" + # TODO We can't merge nonexistent vars + exclude: + - "nonexistent" + todo: + packages: + include: + - "@bar" + - "@baz" + # TODO We can't merge nonexistent vars + exclude: + - "nonexistent" + # Repositories to fetch packages from + repositories: + otk.include: repository/${version}.yaml + # GPG keys to verify packages with + keys: + otk.include: repository/${version}-gpg.yaml + filesystem: + root: + uuid: 6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 + vfs_type: ext4 + path: / + options: defaults + boot: + uuid: 0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 + vfs_type: ext4 + path: /boot + options: defaults + boot-efi: + uuid: 7B77-95E7 + vfs_type: vfat + path: /boot/efi + options: defaults,uid=0,gid=0,umask=077,shortname=winnt + passno: 2 + +otk.target.osbuild: + pipelines: + - otk.include: "osbuild/root.yaml" + - otk.include: "osbuild/pipeline/tree.yaml" + - otk.include: "osbuild/pipeline/raw.yaml" + - otk.include: "osbuild/pipeline/xz.yaml" diff --git a/example/fedora/minimal.yaml b/example/fedora/minimal.yaml deleted file mode 100644 index 859e9fb0..00000000 --- a/example/fedora/minimal.yaml +++ /dev/null @@ -1,59 +0,0 @@ -otk.version: 1 - -otk.include: - path: repositories/${version}.yaml - -otk.define: - # Packages used by the various pipelines, `root` is the buildroot, `tree` is - # the tree that ends up in the image. The buildroot gets used to build the - # other pipelines so it needs the tooling used in those. - packages: - default: - root: - include: - - "@core" - tree: - include: - otk.op.list.join: - - - "@core" - - initial-setup - - libxkbdcommon - - NetworkManager-wifi - - brcmfmac-firmware - - realtek-firmware - - iwlwifi-mvm-firmware - otk.customization.simon: - defined: - - somepackage - filesystems: - raw: - otk.customization.filesystem: - default: - root: - uuid: 6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 - vfs_type: ext4 - path: / - options: defaults - boot: - uuid: 0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 - vfs_type: ext4 - path: /boot - options: defaults - boot-efi: - uuid: 7B77-95E7 - vfs_type: vfat - path: /boot/efi - options: defaults,uid=0,gid=0,umask=077,shortname=winnt - passno: 2 - defined: - otk.external.osbuild.filesystem: ${this} - - -otk.target.osbuild: - pipelines: - - otk.include: osbuild/pipelines/root.yaml - - otk.include: osbuild/pipelines/tree.yaml - - otk.include: osbuild/pipelines/raw.yaml - -otk.meta.osbuild-composer: - pass diff --git a/example/fedora/osbuild/pipeline/image.yaml b/example/fedora/osbuild/pipeline/image.yaml new file mode 100644 index 00000000..4eddb872 --- /dev/null +++ b/example/fedora/osbuild/pipeline/image.yaml @@ -0,0 +1,2 @@ +foo: + bar diff --git a/example/fedora/osbuild/pipeline/raw.yaml b/example/fedora/osbuild/pipeline/raw.yaml new file mode 100644 index 00000000..259026c4 --- /dev/null +++ b/example/fedora/osbuild/pipeline/raw.yaml @@ -0,0 +1,17 @@ +name: raw +source-epoch: 1659397331 +stages: + # - otk.external.osbuild.depsolve-dnf4: + # architecture: ${architecture} + # module_platform_id: f${version} + # repositories: ${repositories} + # gpgkeys: ${gpgkeys} + # exclude: + # docs: true + # packages: ${packages.default.root} + - type: org.osbuild.selinux + options: + file_contexts: etc/selinux/targeted/contexts/files/file_contexts + labels: + /usr/bin/cp: system_u:object_r:install_exec_t:s0 + /usr/bin/tar: system_u:object_r:install_exec_t:s0 diff --git a/example/fedora/osbuild/pipeline/tree.yaml b/example/fedora/osbuild/pipeline/tree.yaml new file mode 100644 index 00000000..37de8708 --- /dev/null +++ b/example/fedora/osbuild/pipeline/tree.yaml @@ -0,0 +1,68 @@ +name: tree +build: name:root +stages: + - otk.external.osbuild.depsolve-dnf4: + architecture: ${architecture} + module_platform_id: f${version} + repositories: ${packages.repositories} + gpgkeys: ${packages.keys} + docs: ${packages.tree.docs} + weak: ${packages.tree.weak} + packages: ${packages.tree.packages} + - type: org.osbuild.kernel-cmdline + options: + root_fs_uuid: ${filesystem.root.uuid} + kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0 + - type: org.osbuild.fix-bls + options: + prefix: '' + - type: org.osbuild.locale + options: + language: + otk.customization.locale: + scope: tree + default: + language: en_US + defined: + language: en_US + - type: org.osbuild.hostname + options: + language: + otk.customization.hostname: + scope: tree + default: + hostname: localhost.localdomain + defined: + language: localhost.localdomain + - type: org.osbuild.timezone + options: + language: + otk.customization.language: + scope: tree + default: + zone: UTC + defined: + zone: UTC + - type: org.osbuild.fstab + options: + filesystems: + - ${filesystem.boot} + - ${filesystem.boot-efi} + - ${filesystem.root} + - type: org.osbuild.grub2 + options: + root_fs_uuid: ${filesystem.root.uuid} + boot_fs_uuid: ${filesystem.boot.uuid} + kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0 + legacy: i386-pc + uefi: + vendor: fedora + unified: true + # TODO: expose this somehow from the depsolve + saved_entry: "ffffffffffffffffffffffffffffffff-${version}" + write_cmdline: false + config: + default: saved + - type: org.osbuild.selinux + options: + file_contexts: etc/selinux/targeted/contexts/files/file_contexts diff --git a/example/fedora/osbuild/pipeline/xz.yaml b/example/fedora/osbuild/pipeline/xz.yaml new file mode 100644 index 00000000..4eddb872 --- /dev/null +++ b/example/fedora/osbuild/pipeline/xz.yaml @@ -0,0 +1,2 @@ +foo: + bar diff --git a/example/fedora/osbuild/pipelines/root.yaml b/example/fedora/osbuild/pipelines/root.yaml deleted file mode 100644 index 1e29a19a..00000000 --- a/example/fedora/osbuild/pipelines/root.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: root -source-epoch: 1659397331 -stages: - - otk.external.osbuild.depsolve-dnf4: - architecture: ${architecture} - module_platform_id: f${version} - repositories: ${repositories} - gpgkeys: ${gpgkeys} - exclude: - docs: true - packages: ${packages.default.root} - - name: org.osbuild.selinux - options: - file_contexts: etc/selinux/targeted/contexts/files/file_contexts - labels: - /usr/bin/cp: system_u:object_r:install_exec_t:s0 - /usr/bin/tar: system_u:object_r:install_exec_t:s0 diff --git a/example/fedora/osbuild/pipelines/tree.yaml b/example/fedora/osbuild/pipelines/tree.yaml deleted file mode 100644 index 2573278f..00000000 --- a/example/fedora/osbuild/pipelines/tree.yaml +++ /dev/null @@ -1,66 +0,0 @@ -name: tree -build: name:root -stages: - - type: org.osbuild.kernel-cmdline - options: - root_fs_uuid: ${filesystems.raw.root.uuid} - kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0 - - otk.external.osbuild.depsolve-dnf4: - architecture: ${architecture} - module_platform_id: f${version} - repositories: ${repositories} - gpgkeys: ${gpgkeys} - exclude: - docs: true - packages: ${packages.default.tree} - - type: org.osbuild.fix-bls - options: - prefix: '' - - otk.customization.locale: - scope: tree # XXX docs - default: - - type: org.osbuild.locale - options: - language: en_US - defined: - - type: org.osbuild.locale - options: - language: ${this.language} - - otk.customization.hostname: - scope: tree - default: - - type: org.osbuild.hostname - options: - hostname: localhost.localdomain - defined: - - type: org.osbuild.hostname - options: - hostname: ${this.hostname} - - otk.customization.timezone: - scope: tree - default: - - type: org.osbuild.timezone - options: - zone: UTC - defined: - - type: org.osbuild.timezone - options: - zone: ${this.timezone} - - type: org.osbuild.fstab - options: ${filesystems.raw} - - type: org.osbuild.grub2 - options: - root_fs_uuid: ${filesystems.raw.root.uuid} - boot_fs_uuid: ${filesystems.raw.boot.uuid} - kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0 - legacy: i386-pc - uefi: - vendor: fedora - unified: true - saved_entry: "ffffffffffffffffffffffffffffffff-${packages.something.kernel-core.version}" - write_cmdline: false - config: - default: saved - - name: org.osbuild.selinux - options: - file_contexts: etc/selinux/targeted/contexts/files/file_contexts diff --git a/example/fedora/osbuild/root.yaml b/example/fedora/osbuild/root.yaml new file mode 100644 index 00000000..6acf059b --- /dev/null +++ b/example/fedora/osbuild/root.yaml @@ -0,0 +1,28 @@ +# `root.yaml` sets up the buildroot for `osbuild`. +name: root +source-epoch: 1659397331 +stages: + - otk.external.osbuild.depsolve-dnf4: + architecture: ${architecture} + module_platform_id: f${version} + docs: ${packages.root.docs} + weak: ${packages.root.weak} + repositories: ${packages.repositories} + gpgkeys: ${packages.keys} + packages: + include: + otk.op.seq.merge: + values: + - ${packages.root.packages.include} + - ${packages.todo.packages.include} + exclude: + otk.op.seq.merge: + values: + - ${packages.root.packages.exclude} + - ${packages.todo.packages.exclude} + - type: org.osbuild.selinux + options: + file_contexts: etc/selinux/targeted/contexts/files/file_contexts + labels: + /usr/bin/cp: system_u:object_r:install_exec_t:s0 + /usr/bin/tar: system_u:object_r:install_exec_t:s0 diff --git a/example/fedora/repositories/38.yaml b/example/fedora/repositories/38.yaml deleted file mode 100644 index cd695cdf..00000000 --- a/example/fedora/repositories/38.yaml +++ /dev/null @@ -1,6 +0,0 @@ -otk.define: - repositories: - - id: "default" - baseurl: "foo" - gpgkeys: - - "1234" diff --git a/example/fedora/repositories/39.yaml b/example/fedora/repositories/39.yaml deleted file mode 100644 index cd695cdf..00000000 --- a/example/fedora/repositories/39.yaml +++ /dev/null @@ -1,6 +0,0 @@ -otk.define: - repositories: - - id: "default" - baseurl: "foo" - gpgkeys: - - "1234" diff --git a/example/fedora/repositories/40.yaml b/example/fedora/repositories/40.yaml deleted file mode 100644 index 70bac4f8..00000000 --- a/example/fedora/repositories/40.yaml +++ /dev/null @@ -1,6 +0,0 @@ -otk.define: - repositories: - - id: "fedora" - metalink: https://mirrors.fedoraproject.org/metalink?repo=fedora-${version}&arch=${architecture} - gpgkeys: - - "1234" diff --git a/example/fedora/repositories/rawhide.yaml b/example/fedora/repositories/rawhide.yaml deleted file mode 100644 index cd695cdf..00000000 --- a/example/fedora/repositories/rawhide.yaml +++ /dev/null @@ -1,6 +0,0 @@ -otk.define: - repositories: - - id: "default" - baseurl: "foo" - gpgkeys: - - "1234" diff --git a/example/fedora/osbuild/pipelines/raw.yaml b/example/fedora/repository/40-gpg.yaml similarity index 100% rename from example/fedora/osbuild/pipelines/raw.yaml rename to example/fedora/repository/40-gpg.yaml diff --git a/example/fedora/repository/40.yaml b/example/fedora/repository/40.yaml new file mode 100644 index 00000000..bb2da379 --- /dev/null +++ b/example/fedora/repository/40.yaml @@ -0,0 +1,2 @@ +- id: "fedora" + metalink: https://mirrors.fedoraproject.org/metalink?repo=fedora-${version}&arch=${architecture} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..10318a24 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "otk" +version = "2024.0.0" +requires-python = ">= 3.9" + +dependencies = [ + "click >= 8.0", + "parsimonious >= 0.10", + "pyyaml >= 6.0", + "rich >= 13.0", +] + +[project.scripts] +otk = "otk.command:root" + +[project.optional-dependencies] +dev = [ + "ruff", + "pytest >= 8.0", + "mypy >= 1.9", + "types-PyYAML >= 6.0", +] + + +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" diff --git a/src/otk/__init__.py b/src/otk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/__main__.py b/src/otk/__main__.py new file mode 100644 index 00000000..43a7e9a4 --- /dev/null +++ b/src/otk/__main__.py @@ -0,0 +1,3 @@ +from .command import root + +root() diff --git a/src/otk/command.py b/src/otk/command.py new file mode 100644 index 00000000..e55ce681 --- /dev/null +++ b/src/otk/command.py @@ -0,0 +1,108 @@ +import logging +import json +import pathlib +import sys + +import click + +from rich.logging import RichHandler +from rich.console import Console + +from .help.log import JSONSequenceHandler +from .parse.document import Omnifest +from .transform import resolve +from .transform.context import Context + + +log = logging.getLogger(__name__) + + +@click.group() +@click.option( + "-v", + "--verbose", + count=True, + help="Sets verbosity. Can be passed multiple times to be more verbose.", +) +@click.option( + "-j/", + "--json/--no-json", + default=False, + help="Sets output format to JSONseq. Output on stderr will be JSONseq records with ASCII record separators.", +) +@click.option( + "-i", + "--identifier", + help="An identifier to include in all log records generated by `otk -j`. Can only be used together with `-j`.", +) +def root(verbose: int, json: bool, identifier: str) -> None: + """`otk` is the omnifest toolkit. A program to work with omnifest inputs + and translate them into the native formats for image build tooling.""" + + logging.basicConfig( + level=logging.WARNING - (10 * verbose), + handlers=[ + ( + JSONSequenceHandler(identifier, stream=sys.stderr) + if json + else RichHandler( + omit_repeated_times=False, + show_path=False, + console=Console(stderr=True), + ) + ) + ], + ) + + # We do this check *after* setting up the handlers so the error is formatted + if identifier and not json: + log.error("cannot use `-i` without also using `-j`") + sys.exit(1) + + +@root.command() +@click.argument("input", type=click.Path(exists=True)) +@click.argument("output", type=click.Path(), required=False) +def compile(input: str, output: str | None) -> None: + """Compile a given omnifest into its targets.""" + + log.info("Compiling the input file %r to %r", input, output) + + +@root.command() +@click.argument("input", type=click.Path(exists=True)) +@click.argument("output", type=click.Path(), required=False) +def flatten(input: str, output: str | None) -> None: + """Flatten a given omnifest by performing all includes.""" + + log.info("flattening %r to %r", input, output or "STDOUT") + + file = pathlib.Path(input) + root = file.parent + + # TODO posixpath serializer for json output + log.info("include root is %r", str(root)) + + context = Context(root) + + tree = Omnifest.from_yaml_path(file).to_tree() + + prev_tree = tree + step_tree = 0 + + while True: + step_tree += 1 + + log.debug("resolve cycle %r", step_tree) + + next_tree = resolve(context, prev_tree) + if prev_tree == next_tree: + break + + prev_tree = next_tree + + tree = next_tree + + log.debug("tree is stable") + + print(json.dumps(tree, indent=2)) diff --git a/src/otk/error.py b/src/otk/error.py new file mode 100644 index 00000000..a801b8d3 --- /dev/null +++ b/src/otk/error.py @@ -0,0 +1,88 @@ +"""Error types used by `otk`.""" + + +class OTKError(Exception): + """Any application raised by `otk` inherits from this.""" + + pass + + +class ParseError(OTKError): + """General base exception for any errors related to parsing omnifests.""" + + pass + + +class ParseTypeError(ParseError): + """A wrong type was encountered for a position in the omnifest.""" + + pass + + +class ParseKeyError(ParseTypeError): + """A required key was missing for a position in the omnifest.""" + + pass + + +class ParseValueError(ParseTypeError): + """A required value was missing for a position in the omnifest.""" + + pass + + +class TransformError(OTKError): + """General base exception for any errors related to transforming + omnifests.""" + + pass + + +class TransformVariableLookupError(TransformError): + """Failed to look up a variable.""" + + pass + + +class TransformVariableTypeError(TransformError): + """Failed to look up a variable due to a parent variable type.""" + + pass + + +class TransformVariableIndexTypeError(TransformError): + """Failed to look up an index into a variable due to the index contents + not being a number.""" + + pass + + +class TransformVariableIndexRangeError(TransformError): + """Failed to look up an index into a variable due to it being out of + bounds.""" + + pass + + +class TransformDefineDuplicateError(TransformError): + """Tried to redefine a variable.""" + + pass + + +class TransformDirectiveTypeError(TransformError): + """A directive received the wrong types.""" + + pass + + +class TransformDirectiveArgumentError(TransformError): + """A directive received the wrong argument(s).""" + + pass + + +class TransformDirectiveUnknownError(TransformError): + """Unknown directive.""" + + pass diff --git a/src/otk/help/__init__.py b/src/otk/help/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/help/log.py b/src/otk/help/log.py new file mode 100644 index 00000000..f4b072f5 --- /dev/null +++ b/src/otk/help/log.py @@ -0,0 +1,37 @@ +import logging +import json + + +class _JSONFormatter(logging.Formatter): + def __init__( + self, + identifier: str | None = None, + ) -> None: + super().__init__() + self.identifier = identifier + + def format(self, record: logging.LogRecord) -> str: + if self.identifier: + if hasattr(record, "identifier"): + raise ValueError("identifier already in log record") + + record.identifier = self.identifier + + return json.dumps(record.__dict__) + + +class JSONSequenceHandler(logging.StreamHandler): + def __init__(self, identifier: str | None = None, stream=None) -> None: + super().__init__(stream) + + self.formatter = _JSONFormatter(identifier) + + def emit(self, record: logging.LogRecord) -> None: + self.acquire() + + self.stream.write("\x1e") + self.stream.write(self.format(record)) + self.stream.write("\n") + self.flush() + + self.release() diff --git a/src/otk/parse/__init__.py b/src/otk/parse/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/parse/document.py b/src/otk/parse/document.py new file mode 100644 index 00000000..f83ef10c --- /dev/null +++ b/src/otk/parse/document.py @@ -0,0 +1,74 @@ +import pathlib +import logging + +import yaml + +from typing import Self, Any + +from ..error import ParseTypeError, ParseKeyError + + +log = logging.getLogger(__name__) + + +class Omnifest: + # Underlying data is a dictionary containing the validated deserialized + # contents of the bytes that were used to create this omnifest. + _underlying_data: dict[str, Any] + + def __init__(self, underlying_data: dict[str, Any]) -> None: + self._underlying_data = underlying_data + + @classmethod + def from_yaml_bytes(cls, text: bytes, ensure: bool = True) -> Self: + deserialized_data = cls.read(yaml.safe_load(text), ensure) + + if ensure: + cls.ensure(deserialized_data) + + return cls(deserialized_data) + + @classmethod + def from_yaml_path(cls, path: pathlib.Path, ensure: bool = True) -> Self: + """Read a YAML file into an Omnifest instance.""" + + log.debug("reading yaml from path %r", str(path)) + + # This is an invariant that should be handled at the calling side of + # this function + assert path.exists(), "path to exist" + + with path.open("rb") as file: + return cls.from_yaml_bytes(file.read(), ensure) + + @classmethod + def read(cls, deserialized_data: Any, ensure: bool = True) -> dict[str, Any]: + """Take any type returned by a deserializer and ensure that it is + something that could represent an Omnifest.""" + + if ensure: + # The top level of the Omnifest needs to be a dictionary + if not isinstance(deserialized_data, dict): + log.error( + "data did not deserialize to a dictionary: type=%r,data=%r", + type(deserialized_data), + deserialized_data, + ) + raise ParseTypeError("omnifest must deserialize to a dictionary") + + return deserialized_data + + @classmethod + def ensure(cls, deserialized_data: dict[str, Any]) -> None: + """Take a dictionary and ensure that its keys and values would be + considered an Omnifest.""" + + # And that dictionary needs to contain certain keys to indicate this + # being an Omnifest. + if "otk.version" not in deserialized_data: + raise ParseKeyError( + "omnifest must contain a key by the name of `otk.version`" + ) + + def to_tree(self) -> dict[str, Any]: + return self._underlying_data diff --git a/src/otk/parse/schema/__init__.py b/src/otk/parse/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/parse/schema/omnifest.py b/src/otk/parse/schema/omnifest.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/parse/token/__init__.py b/src/otk/parse/token/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/parse/token/name.py b/src/otk/parse/token/name.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/parse/token/variable.py b/src/otk/parse/token/variable.py new file mode 100644 index 00000000..e69de29b diff --git a/src/otk/transform/__init__.py b/src/otk/transform/__init__.py new file mode 100644 index 00000000..e65e92b5 --- /dev/null +++ b/src/otk/transform/__init__.py @@ -0,0 +1,124 @@ +"""Transforming trees.""" + +import logging + +from typing import Any, Type + +from .context import Context +from .directive import op, define, include, desugar, customization + + +log = logging.getLogger(__name__) + + +def resolve_dict(ctx: Context, tree: dict[str, Any]) -> Any: + log.debug("resolving dict %r", tree) + + for key, val in tree.items(): + if key == "otk.include": + data = include(ctx, val) + + # We special case dictionaries of length one, when they contain a directive + # it means we want to replace the entire dictionary with the value of that + # directive + if len(tree) == 1: + log.debug( + "single-item otk.include dict %r replacing with %r", + tree, + data, + ) + return resolve(ctx, data) + else: + # This seems like we're ignoring the rest of the keys, however + # since resolving happens in multiple cycles those keys will + # get resolved on the next cycle. + log.debug( + "multi-item otk.include dict %r updating with %r", + tree, + data, + ) + return tree | resolve(ctx, data) + + if key.startswith("otk.op"): + # TODO Do we want to pass key always? + # TODO Is this the correct order? + data = op(ctx, resolve(ctx, val), key) + + # We special case dictionaries of length one, when they contain a directive + # it means we want to replace the entire dictionary with the value of that + # directive + if len(tree) == 1: + log.debug("single-item otk.op dict %r replacing with %r", tree, data) + return resolve(ctx, data) + else: + # This seems like we're ignoring the rest of the keys, however + # since resolving happens in multiple cycles those keys will + # get resolved on the next cycle. + log.debug("multi-item otk.op dict %r updating with %r", tree, data) + return tree | resolve(ctx, data) + + if key == "otk.define": + define(ctx, val) + + if key.startswith("otk.customization."): + return resolve(ctx, customization(ctx, val, key)) + + tree[key] = resolve(ctx, val) + + return tree + + +def resolve_list(ctx, tree: list[Any]) -> list[Any]: + log.debug("resolving list %r", tree) + + for idx, val in enumerate(tree): + tree[idx] = resolve(ctx, val) + + return tree + + +def resolve_str(ctx, tree: str) -> Any: + log.debug("resolving str %r", tree) + return desugar(ctx, tree) + + +def resolve_int(ctx, tree: int) -> int: + log.debug("resolving int %r", tree) + return tree + + +def resolve_float(ctx, tree: float) -> float: + log.debug("resolving float %r", tree) + return tree + + +def resolve_bool(ctx, tree: bool) -> bool: + log.debug("resolving bool %r", tree) + return tree + + +def resolve_none(ctx: Context, tree: None) -> None: + log.debug("resolving none %r", tree) + return tree + + +resolvers: dict[Type, Any] = { + dict: resolve_dict, + list: resolve_list, + str: resolve_str, + int: resolve_int, + float: resolve_float, + bool: resolve_bool, + type(None): resolve_none, +} + + +def resolve(ctx: Context, tree: Any) -> Any: + """Resolves a (sub)tree of any type into a new tree. Each type has its own + specific handler to rewrite the tree.""" + + if type(tree) not in resolvers: + log.fatal("could not look up %r in resolvers", type(tree)) + raise Exception(type(tree)) + + return resolvers[type(tree)](ctx, tree) diff --git a/src/otk/transform/context.py b/src/otk/transform/context.py new file mode 100644 index 00000000..09b0e792 --- /dev/null +++ b/src/otk/transform/context.py @@ -0,0 +1,76 @@ +# Enables postponed annotations on older snakes (PEP-563) +# Enables | union syntax for types on older snakes (PEP-604) +from __future__ import annotations + +import logging +import pathlib + +from typing import Any + +from ..error import ( + TransformVariableLookupError, + TransformVariableTypeError, + TransformDefineDuplicateError, + TransformVariableIndexTypeError, + TransformVariableIndexRangeError, +) + + +log = logging.getLogger(__name__) + + +class Context: + _version: int | None + _path: pathlib.Path + _variables: dict[str, Any] + + def __init__(self, path: pathlib.Path | None = None) -> None: + self._version = None + self._path = path if path else pathlib.Path(".") + self._variables = {} + + def version(self, v: int) -> None: + # Set the context version, duplicate definitions with different + # versions are an error + if self._version is not None: + if self._version != v: + raise ValueError("duplicate but different version") + + log.debug("context setting version to %r", v) + self._version = v + + def define(self, name: str, value: Any) -> None: + # Since we go through the tree multiple times it's easy to ignore + # duplicate definitions as long as they define to the *same* value. + if name in self._variables and self._variables[name] != value: + raise TransformDefineDuplicateError() + + self._variables[name] = value + + def variable(self, name: str) -> Any: + parts = name.split(".") + + value = self._variables + + for part in parts: + if isinstance(value, dict): + if part not in value: + raise TransformVariableLookupError("Could not find %r" % parts) + + # TODO how should we deal with integer keys, convert them + # TODO on KeyError? Check for existence of both? + value = value[part] + elif isinstance(value, list): + if not part.isnumeric(): + raise TransformVariableIndexTypeError() + + try: + value = value[int(part)] + except IndexError: + raise TransformVariableIndexRangeError() + else: + raise TransformVariableTypeError( + "tried to look up %r.%r but %r isn't a dictionary" + ) + + return value diff --git a/src/otk/transform/directive.py b/src/otk/transform/directive.py new file mode 100644 index 00000000..2e2e533a --- /dev/null +++ b/src/otk/transform/directive.py @@ -0,0 +1,170 @@ +"""Implements the directives, these are named transformations that can be used +in an omnifest.""" + +# Enables postponed annotations on older snakes (PEP-563) +# Enables | union syntax for types on older snakes (PEP-604) +from __future__ import annotations + +import itertools +import logging +import pathlib + +from typing import Any + +from .context import Context +from ..parse.document import Omnifest +from ..error import ( + TransformDirectiveTypeError, + TransformDirectiveUnknownError, +) +from .. import tree + + +log = logging.getLogger(__name__) + + +# The prefix used for directives +PREFIX = "otk." + + +@tree.must_be(dict) +def define(ctx: Context, tree: Any) -> Any: + """Takes an `otk.define` block (which must be a dictionary and registers + everything in it as variables in the context.""" + + for key, value in tree.items(): + ctx.define(key, value) + + +@tree.must_be(str) +def include(ctx: Context, tree: Any) -> Any: + """Include a separate file.""" + + tree = desugar(ctx, tree) + + file = ctx._path / pathlib.Path(tree) + + # TODO str'ed for json log, lets add a serializer for posixpath + # TODO instead + log.info("otk.include=%s", str(file)) + + if not file.exists(): + # TODO, better error type + raise Exception("otk.include nonexistent file %r" % file) + + return Omnifest.from_yaml_path(file, ensure=False).to_tree() + + +def op(ctx: Context, tree: Any, key: str) -> Any: + """Dispatch the various `otk.op` directives while handling unknown + operations.""" + + if key == "otk.op.seq.merge": + return op_seq_merge(ctx, tree) + elif key == "otk.op.map.merge": + return op_map_merge(ctx, tree) + else: + raise TransformDirectiveUnknownError("nonexistent op %r", key) + + +@tree.must_be(dict) +@tree.must_pass(tree.has_keys(["values"])) +def op_seq_merge(ctx: Context, tree: dict[str, Any]) -> Any: + """Merge to sequences by concatenating them together.""" + + values = tree["values"] + + if not isinstance(values, list): + raise TransformDirectiveTypeError( + "seq merge received values of the wrong type, was expecting a list of lists but got %r", + values, + ) + + if not all(isinstance(sl, list) for sl in values): + raise TransformDirectiveTypeError( + "seq merge received values of the wrong type, was expecting a list of lists but got %r", + values, + ) + + return list(itertools.chain.from_iterable(values)) + + +@tree.must_be(dict) +@tree.must_pass(tree.has_keys(["values"])) +def op_map_merge(ctx: Context, tree: dict[str, Any]) -> Any: + """Merge two dictionaries. Keys from the second dictionary overwrite keys + in the first dictionary.""" + + values = tree["values"] + + if not isinstance(values, list): + raise TransformDirectiveTypeError( + "map merge received values of the wrong type, was expecting a list of lists but got %r", + values, + ) + + if not all(isinstance(sl, dict) for sl in values): + raise TransformDirectiveTypeError( + "map merge received values of the wrong type, was expecting a list of dicts but got %r", + values, + ) + + result = {} + + for value in values: + result.update(value) + + return result + + +@tree.must_be(dict) +@tree.must_pass(tree.has_keys(["default", "scope", "defined"])) +def customization(ctx: Context, tree: dict[str, Any], key) -> Any: + """Apply a customization.""" + log.warning("applying customization %r", key) + + # TODO take in customizations somewhere and use there here + return tree["default"] + + +@tree.must_be(str) +def desugar(ctx: Context, tree: str) -> Any: + """Desugar a string. If the string consists of a single `${name}` value + then we return the object it refers to by looking up its name in the + variables. + + If the string has anything around a variable such as `foo${name}-${bar}` + then we replace the values inside the string. This requires the type of + the variable to be replaced to be either str, int, or float.""" + + # TODO use parsimonous instead of this + + if tree.startswith("${"): + name = tree[2 : tree.index("}")] + data = ctx.variable(tree[2 : tree.index("}")]) + log.debug("desugaring %r as fullstring to %r", name, data) + return ctx.variable(tree[2 : tree.index("}")]) + + if "${" in tree: + name = tree[tree.index("${") + 2 : tree.index("}")] + head = tree[: tree.index("${")] + tail = tree[tree.index("}") + 1 :] + + data = ctx.variable(name) + + if isinstance(data, (int, float)): + data = str(data) + + if not isinstance(data, str): + raise TransformDirectiveTypeError( + "string sugar resolves to an incorrect type, expected int, float, or str but got %r", + data, + ) + + data = head + data + tail + + log.debug("desugaring %r as substring to %r", name, data) + + return data + + return tree diff --git a/src/otk/tree.py b/src/otk/tree.py new file mode 100644 index 00000000..1d296785 --- /dev/null +++ b/src/otk/tree.py @@ -0,0 +1,57 @@ +"""`otk` is primarily a tree transformation tool. This module contains the +objects used to work with trees.""" + +# Enables postponed annotations on older snakes (PEP-563) +# Enables | union syntax for types on older snakes (PEP-604) +from __future__ import annotations + +import functools + +from typing import Type + +from .error import ( + TransformDirectiveTypeError, + TransformDirectiveArgumentError, +) + + +def must_be(kind: Type): + """Handles the tree having to be of a specific type at runtime.""" + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + if not isinstance(args[1], kind): + raise TransformDirectiveTypeError( + "otk.define expects a %r as its argument but received a `%s`: `%r`" + % (kind, type(args[1]), args[1]) + ) + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def must_pass(*vs): + """Handles the tree having to pass specific validators at runtime.""" + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + for v in vs: + v(args[1]) + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def has_keys(keys): + def inner(tree): + for key in keys: + if key not in tree: + raise TransformDirectiveArgumentError("Expected key %r", key) + + return inner diff --git a/test/test_parse.py b/test/test_parse.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_parse_document.py b/test/test_parse_document.py new file mode 100644 index 00000000..4ac86b14 --- /dev/null +++ b/test/test_parse_document.py @@ -0,0 +1,66 @@ +import pytest + +from otk.parse.document import Omnifest +from otk.error import ParseTypeError, ParseKeyError + + +def test_omnifest_read_dict(): + assert Omnifest.read({}) == {} + + +def test_omnifest_read_list(): + with pytest.raises(ParseTypeError): + Omnifest.read([]) + + +def test_omnifest_read_string(): + with pytest.raises(ParseTypeError): + Omnifest.read("") + + +def test_omnifest_ensure(): + Omnifest.ensure({"otk.version": 1}) + + +def test_omnifest_ensure_no_keys(): + with pytest.raises(ParseKeyError): + assert Omnifest.ensure({}) + + +def test_omnifest_from_yaml_bytes(): + # Happy + Omnifest.from_yaml_bytes(""" +otk.version: 1 +""") + + # A bunch of YAMLs that don't contain a top level map + with pytest.raises(ParseTypeError): + Omnifest.from_yaml_bytes(""" +""") + + with pytest.raises(ParseTypeError): + Omnifest.from_yaml_bytes(""" +1 +""") + + with pytest.raises(ParseTypeError): + Omnifest.from_yaml_bytes(""" +- 1 +- 2 +""") + + with pytest.raises(ParseTypeError): + Omnifest.from_yaml_bytes(""" +"1" +""") + + # And a YAML that does have a top level map, but no `otk.version` key + with pytest.raises(ParseKeyError): + Omnifest.from_yaml_bytes(""" +otk.not-version: 1 +""") + + +def test_omnifest_from_yaml_path(): + # TODO + ... diff --git a/test/test_transform_context.py b/test/test_transform_context.py new file mode 100644 index 00000000..c85e7244 --- /dev/null +++ b/test/test_transform_context.py @@ -0,0 +1,65 @@ +import pytest + +from otk.transform.context import Context +from otk.error import ( + TransformVariableLookupError, + TransformVariableTypeError, + TransformVariableIndexTypeError, + TransformVariableIndexRangeError, +) + + +def test_context(): + ctx = Context() + ctx.define("foo", "foo") + + assert ctx.variable("foo") == "foo" + + ctx.define("bar", {"bar": "foo"}) + + assert ctx.variable("bar.bar") == "foo" + + ctx.define("baz", {"baz": {"baz": "foo", "0": 1, 1: "foo"}}) + + assert ctx.variable("baz.baz.baz") == "foo" + assert ctx.variable("baz.baz.0") == 1 + + # TODO numeric key lookups! + # assert ctx.variable("baz.baz.1") == "foo" + + ctx.define("boo", [1, 2]) + + assert ctx.variable("boo") == [1, 2] + assert ctx.variable("boo.0") == 1 + assert ctx.variable("boo.1") == 2 + + +def test_context_nonexistent(): + ctx = Context() + + with pytest.raises(TransformVariableLookupError): + ctx.variable("foo") + + with pytest.raises(TransformVariableLookupError): + ctx.variable("foo.bar") + + ctx.define("bar", {"bar": "foo"}) + + with pytest.raises(TransformVariableLookupError): + ctx.variable("bar.nonexistent") + + +def test_context_unhappy(): + ctx = Context() + ctx.define("foo", "foo") + + with pytest.raises(TransformVariableTypeError): + ctx.variable("foo.bar") + + ctx.define("bar", ["bar"]) + + with pytest.raises(TransformVariableIndexTypeError): + ctx.variable("bar.bar") + + with pytest.raises(TransformVariableIndexRangeError): + ctx.variable("bar.3") diff --git a/test/test_transform_directive.py b/test/test_transform_directive.py new file mode 100644 index 00000000..2b68e817 --- /dev/null +++ b/test/test_transform_directive.py @@ -0,0 +1,111 @@ +import pytest + +from otk.transform.directive import op_seq_merge, op_map_merge, desugar, define, include +from otk.transform.context import Context +from otk.error import TransformDirectiveTypeError, TransformDirectiveArgumentError + + +def test_define(): + ctx = Context() + + define(ctx, {"a": "b", "c": 1}) + + assert ctx.variable("a") == "b" + assert ctx.variable("c") == 1 + + +def test_define_unhappy(): + ctx = Context() + + with pytest.raises(TransformDirectiveTypeError): + define(ctx, 1) + + with pytest.raises(TransformDirectiveTypeError): + define(ctx, "str") + + +def test_include_unhappy(): + ctx = Context() + + with pytest.raises(TransformDirectiveTypeError): + include(ctx, 1) + + +def test_op_seq_merge(): + ctx = Context() + + l1 = [1, 2, 3] + l2 = [4, 5, 6] + + d = {"values": [l1, l2]} + + assert op_seq_merge(ctx, d) == [1, 2, 3, 4, 5, 6] + + +def test_op_seq_merge_unhappy(): + ctx = Context() + + with pytest.raises(TransformDirectiveTypeError): + op_seq_merge(ctx, 1) + + with pytest.raises(TransformDirectiveArgumentError): + op_seq_merge(ctx, {}) + + with pytest.raises(TransformDirectiveTypeError): + op_seq_merge(ctx, {"values": 1}) + + with pytest.raises(TransformDirectiveTypeError): + op_seq_merge(ctx, {"values": [1, {2: 3}]}) + + +def test_op_map_merge(): + ctx = Context() + + d1 = {"foo": "bar"} + d2 = {"bar": "foo"} + + d = {"values": [d1, d2]} + + assert op_map_merge(ctx, d) == {"foo": "bar", "bar": "foo"} + + +def test_op_map_merge_unhappy(): + ctx = Context() + + with pytest.raises(TransformDirectiveTypeError): + op_map_merge(ctx, 1) + + with pytest.raises(TransformDirectiveArgumentError): + op_map_merge(ctx, {}) + + with pytest.raises(TransformDirectiveTypeError): + op_map_merge(ctx, {"values": 1}) + + with pytest.raises(TransformDirectiveTypeError): + op_map_merge(ctx, {"values": [1, {2: 3}]}) + + +def test_desugar(): + ctx = Context() + ctx.define("str", "bar") + ctx.define("int", 1) + ctx.define("float", 1.1) + + assert desugar(ctx, "") == "" + assert desugar(ctx, "${str}") == "bar" + assert desugar(ctx, "a${str}b") == "abarb" + assert desugar(ctx, "${int}") == 1 + assert desugar(ctx, "a${int}b") == "a1b" + assert desugar(ctx, "${float}") == 1.1 + assert desugar(ctx, "a${float}b") == "a1.1b" + + +def test_desugar_unhappy(): + ctx = Context() + ctx.define("dict", {}) + + with pytest.raises(TransformDirectiveTypeError): + desugar(ctx, 1) + + with pytest.raises(TransformDirectiveTypeError): + desugar(ctx, "a${dict}b") diff --git a/test/test_tree.py b/test/test_tree.py new file mode 100644 index 00000000..e69de29b