CWL definitions

.github/workflows/tests.yml

@@ -0,0 +1,67 @@
+# run test suites
+
+name: Tests
+on:
+  - pull_request
+  - push
+  - release
+  - workflow_dispatch
+
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # see: https://github.com/fkirc/skip-duplicate-actions
+  skip_duplicate:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_duplicate.outputs.should_skip && ! contains(github.ref, 'refs/tags') && ! contains(github.ref, 'refs/heads/main') }}
+    steps:
+      - uses: fkirc/skip-duplicate-actions@master
+        with:
+          concurrent_skipping: "same_content_newer"
+          skip_after_successful_duplicate: "true"
+          cancel_others: "true"
+          do_not_skip: '["workflow_dispatch", "schedule", "release"]'
+
+  # see: https://github.com/actions/setup-python
+  tests:
+    needs: skip_duplicate
+    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        test-case: ["test-only"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: 'pip'
+      - name: Parse Python Version
+        id: python-semver
+        run: |
+          echo "::set-output name=major:$(echo ${{ matrix.python-version }} | cut -d '.' -f 1)"
+          echo "::set-output name=minor:$(echo ${{ matrix.python-version }} | cut -d '.' -f 2)"
+      - name: Install Dependencies
+        run: make install-dev
+      - name: Display Packages
+        run: |
+          pip freeze
+          make info
+      - name: Display Environment Variables
+        run: |
+          hash -r
+          env | sort
+      - name: Run Tests
+        run: make ${{ matrix.test-case }}

.gitignore

@@ -0,0 +1,177 @@
+### VisualStudioCode
+.vscode/
+# Local History for Visual Studio Code
+.history/
+# Built Visual Studio Code Extensions
+*.vsix
+
+### JetBrains
+# User-specific stuff
+.idea/
+.run/
+
+### VirtualEnv
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+
+### Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### Output Files
+*.cwl
+!src/goldfinch/processes/**/*.cwl
+*.y[a]ml
+*.json
+!src/goldfinch/processes/subset/small_geojson.json

Makefile

@@ -0,0 +1,77 @@
+# Included custom configs change the value of MAKEFILE_LIST
+# Extract the required reference beforehand so we can use it for help target
+MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+# Include custom config if it is available
+-include Makefile.config
+
+APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..)
+APP_NAME := $(shell basename $(APP_ROOT))
+APP_VERSION := 0.1.0
+APP_DOCKER_VERSION := ghcr.io/bird-house/$(APP_NAME)
+APP_DOCKER_REGISTRY := ghcr.io/bird-house/$(APP_NAME)
+APP_DOCKER_IMAGE := $(APP_DOCKER_REGISTRY):$(APP_VERSION)
+
+filter_out_substr = $(foreach v,$(2),$(if $(findstring $(1),$(v)),,$(v)))
+CWL_CLI_SOURCES := $(wildcard $(APP_ROOT)/src/$(APP_NAME)/processes/*/*.py)
+CWL_CLI_SOURCES := $(call filter_out_substr,test, $(CWL_CLI_SOURCES))
+CWL_CLI_OUTPUTS := $(CWL_CLI_SOURCES:.py=.cwl)
+
+.PHONY: help
+help:
+	@echo "Available targets:"
+	@echo "  help               - Show this help message"
+	@echo "  info               - Display computed variables"
+	@echo "  install            - Install dependencies"
+	@echo "  cwl-generate       - Generate CWL files from Python processes"
+	@echo "  cwl-generate-only  - Generate CWL files without installing dependencies"
+
+.PHONY: info
+info:
+	@echo "APP_VERSION:     [" $(APP_VERSION) "]"
+	@echo "APP_ROOT:        [" $(APP_ROOT) "]"
+	@echo "CWL_CLI_SOURCES: [" $(CWL_CLI_SOURCES) "]"
+	@echo "CWL_CLI_OUTPUTS: [" $(CWL_CLI_OUTPUTS) "]"
+
+.PHONY: install
+install:
+	@echo "Installing dependencies..."
+	@pip install ".[processes]"
+
+.PHONY: install-dev
+install-dev: install
+	@echo "Installing development dependencies..."
+	@pip install ".[dev,processes]"
+
+.PHONY: test-ony
+test-only:
+	@echo "Running tests..."
+	@pytest "$(APP_ROOT)"
+
+.PHONY: test
+test: install-dev test-only
+
+# For each Python file, generate the corresponding CWL file
+# Will only run modified Python files by default
+%.cwl: %.py
+	@echo "Generating CWL for [$<]..."
+	click2cwl \
+		--process $< \
+		--output $@ \
+		--docker "$(APP_DOCKER_IMAGE)" \
+		--cwl-version v1.2 \
+		--metadata "id=$(@F:.cwl=)"
+
+.PHONY: cwl-generate-only
+cwl-generate-only: $(CWL_CLI_OUTPUTS)
+
+.PHONY: cwl-generate
+cwl-generate: cwl-generate-only | install
+
+.PHONY: cwl-generate-all
+cwl-generate-all:
+	$(MAKE) $(foreach file,$(CWL_CLI_SOURCES),-W $(file)) cwl-generate-only
+
+.PHONY: docker-build
+docker-build:
+	@echo "Building Docker image..."
+	docker build -t "$(APP_DOCKER_IMAGE)" -f "$(APP_ROOT)/docker/Dockerfile" "$(APP_ROOT)"

README.md

@@ -1,2 +1,58 @@
 # goldfinch
-Climate service processes defined through CWL
+
+Climate service processes defined through CWL.
+
+## Setup and Development
+
+For quick setup only with specific CWL dependencies, install the package as follows.
+
+```shell
+pip install -e .
+```
+
+Install the specific dependencies of the relevant CLI to be generated.
+Otherwise, the following command can be used to install *all* dependencies across all locally defined CLI.
+
+> [!WARNING]
+> Because the CLI tool attempts to load the other Python file to identify its CWL definition from `click` decorators,
+> any packages or dependencies this Python script imports has to be installed in the environment where the tool is run.
+
+```shell
+pip install -e ".[processes]"
+```
+
+## Usage Commands
+
+All options can be obtained by running the following command:
+
+```shell
+click2cwl [-h|--help]
+```
+
+The typical command to run the CWL convertion is as follows for the desired Python CLI process:
+
+```shell
+click2cwl --process <path/to/python.py> [--output-cwl <path/to/package.cwl>] [--output-format <yaml|json>]
+```
+
+By default, the output CWL is produced as YAML on the standard output with only the `CommandLineTool` definition.
+A file can be generated in YAML or JSON format by specifying the `--output-cwl` option.
+
+An embedded `CommandLineTool` within a fully defined CWL `Workflow` graph can be generated by
+specifying the `-w|--workflow` shortcut option or explicitly using the `--cwl <clt|cwl>` options.
+This can also be generated as JSON or YAML, and either written to a file or standard output.
+
+Alternatively, the Job Parameters (which would be used to call the CWL process, see below) can be generated as follows.
+Note that to avoid parameter mangling between the CWL converter and the underlying CLI process arguments, they should
+be distinguished by an explicit `--` separator. This can also use similar output file and format options as above.
+
+```shell
+click2cwl --process <path/to/python.py> --job [<path/to/job-params.yml>] -- [CLI ARGUMENTS ...]
+```
+
+It should then be possible to test the result as follows if everything was invoked correctly
+and with all input requirements met:
+
+```shell
+cwltool <path/to/output.cwl> <path/to/job-params.yml>
+```