CWL definitions

.gitignore

@@ -0,0 +1,177 @@
+### VisualStudioCode
+.vscode/
+# Local History for Visual Studio Code
+.history/
+# Built Visual Studio Code Extensions
+*.vsix
+
+### JetBrains
+# User-specific stuff
+.idea/
+.run/
+
+### VirtualEnv
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+
+### Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### Output Files
+*.cwl
+!src/goldfinch/processes/**/*.cwl
+*.y[a]ml
+*.json
+!src/goldfinch/processes/subset/small_geojson.json

Makefile

@@ -0,0 +1,59 @@
+# Included custom configs change the value of MAKEFILE_LIST
+# Extract the required reference beforehand so we can use it for help target
+MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+# Include custom config if it is available
+-include Makefile.config
+
+APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..)
+APP_NAME := $(shell basename $(APP_ROOT))
+APP_VERSION := 0.1.0
+APP_DOCKER_VERSION := ghcr.io/bird-house/$(APP_NAME)
+APP_DOCKER_REGISTRY := ghcr.io/bird-house/$(APP_NAME)
+APP_DOCKER_IMAGE := $(APP_DOCKER_REGISTRY):$(APP_VERSION)
+
+filter_out_substr = $(foreach v,$(2),$(if $(findstring $(1),$(v)),,$(v)))
+CWL_CLI_SOURCES := $(wildcard $(APP_ROOT)/src/$(APP_NAME)/processes/*/*.py)
+CWL_CLI_SOURCES := $(call filter_out_substr,test, $(CWL_CLI_SOURCES))
+CWL_CLI_OUTPUTS := $(CWL_CLI_SOURCES:.py=.cwl)
+
+.PHONY: help
+help:
+	@echo "Available targets:"
+	@echo "  help               - Show this help message"
+	@echo "  info               - Display computed variables"
+	@echo "  install            - Install dependencies"
+	@echo "  cwl-generate       - Generate CWL files from Python processes"
+	@echo "  cwl-generate-only  - Generate CWL files without installing dependencies"
+
+.PHONY: info
+info:
+	@echo "APP_VERSION:     [" $(APP_VERSION) "]"
+	@echo "APP_ROOT:        [" $(APP_ROOT) "]"
+	@echo "CWL_CLI_SOURCES: [" $(CWL_CLI_SOURCES) "]"
+	@echo "CWL_CLI_OUTPUTS: [" $(CWL_CLI_OUTPUTS) "]"
+
+.PHONY: install
+install:
+	@echo "Installing dependencies..."
+	@pip install ".[processes]"
+
+# For each Python file, generate the corresponding CWL file
+%.cwl: %.py
+	@echo "Generating CWL for [$<]..."
+	click2cwl \
+		--process $< \
+		--output $@ \
+		--docker "$(APP_DOCKER_IMAGE)" \
+		--cwl-version v1.2 \
+		--metadata "id=$(@F:.cwl=)"
+
+.PHONY: cwl-generate-only
+cwl-generate-only: $(CWL_CLI_OUTPUTS)
+
+.PHONY: cwl-generate
+cwl-generate: cwl-generate-only | install
+
+.PHONY: docker-build
+docker-build:
+	@echo "Building Docker image..."
+	docker build -t "$(APP_DOCKER_IMAGE)" -f "$(APP_ROOT)/docker/Dockerfile" "$(APP_ROOT)"

README.md

@@ -1,2 +1,58 @@
 # goldfinch
-Climate service processes defined through CWL
+
+Climate service processes defined through CWL.
+
+## Setup and Development
+
+For quick setup only with specific CWL dependencies, install the package as follows.
+
+```shell
+pip install -e .
+```
+
+Install the specific dependencies of the relevant CLI to be generated.
+Otherwise, the following command can be used to install *all* dependencies across all locally defined CLI.
+
+> [!WARNING]
+> Because the CLI tool attempts to load the other Python file to identify its CWL definition from `click` decorators,
+> any packages or dependencies this Python script imports has to be installed in the environment where the tool is run.
+
+```shell
+pip install -e ".[processes]"
+```
+
+## Usage Commands
+
+All options can be obtained by running the following command:
+
+```shell
+click2cwl [-h|--help]
+```
+
+The typical command to run the CWL convertion is as follows for the desired Python CLI process:
+
+```shell
+click2cwl --process <path/to/python.py> [--output <path/to/package.cwl>] [--output-format <yaml|json>]
+```
+
+By default, the output CWL is produced as YAML on the standard output with only the `CommandLineTool` definition.
+A file can be generated in YAML or JSON format by specifying the `--output` option.
+
+An embedded `CommandLineTool` within a fully defined CWL `Workflow` graph can be generated by
+specifying the `-w|--workflow` shortcut option or explicitly using the `--cwl <clt|cwl>` options.
+This can also be generated as JSON or YAML, and either written to a file or standard output.
+
+Alternatively, the Job Parameters (which would be used to call the CWL process, see below) can be generated as follows.
+Note that to avoid parameter mangling between the CWL converter and the underlying CLI process arguments, they should
+be distinguished by an explicit `--` separator. This can also use similar output file and format options as above.
+
+```shell
+click2cwl --process <path/to/python.py> --job [<path/to/job-params.yml>] -- [CLI ARGUMENTS ...]
+```
+
+It should then be possible to test the result as follows if everything was invoked correctly
+and with all input requirements met:
+
+```shell
+cwltool <path/to/output.cwl> <path/to/job-params.yml>
+```

docker/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+LABEL description.short="Goldfinch processes for climate analysis."
+LABEL description.long="Docker image employed by goldfinch climate processes defined through CWL for portability."
+LABEL maintainer="Francis Charette-Migneault <[email protected]>"
+LABEL vendor="CRIM"
+LABEL version="0.1.0"
+
+COPY src/ ./src/
+COPY Makefile pyproject.toml ./
+RUN apt-get update && \
+    apt-get -y install git make && \
+    make install && \
+    rm Makefile pyproject.toml && \
+    apt-get remove -y build-essential make git && \
+    apt autoremove -y
+
+# should be overridden by the invoked CWL process
+ENTRYPOINT ["bash"]

pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=78.1.1", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "goldfinch"
+version = "0.1.0"
+description = "Climate service processes defined through CWL."
+authors = [
+    { name = "Francis Charette-Migneault", email = "[email protected]" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+dependencies = [
+    "click",
+    "click-option-group",
+    # Temporarily use fork until PR is merged upstream (https://github.com/Terradue/click2cwl/pull/4)
+    # "click2cwl @ git+https://github.com/Terradue/click2cwl"
+    "click2cwl @ git+https://github.com/Terradue/click2cwl.git#egg=fix-path-not-dir"
+]
+
+[project.scripts]
+click2cwl = "goldfinch.utils.click2cwl_cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.optional-dependencies]
+# any additional dependencies needed by underlying processes
+# this is only to facilitate installation "all-in-one"
+processes = [
+    "clisops",
+    "geopandas",
+    "xclim",
+    "h5netcdf",
+]

src/goldfinch/__init__.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+# TODO <will genereate description of process>
+"""
+import argparse
+import os
+import sys
+
+
+def make_parser():
+    name = os.path.splitext(os.path.split(__file__)[-1])[0]
+    ap = argparse.ArgumentParser(prog=name, description=__doc__, add_help=True)
+    # TODO
+    # ap.add_argument("arg_name", help="...")
+    return ap
+
+
+def run(arg_name):
+    return -1
+
+
+def main():
+    ap = make_parser()
+    argv = None if sys.argv[1:] else ['--help']  # auto-help message if no args
+    args = ap.parse_args(args=argv)
+    return run(**vars(args))  # auto-map arguments by name
+
+
+if __name__ == "__main__":
+    main()

src/goldfinch/processes/indicator/environment.yml

@@ -0,0 +1,4 @@
+dependencies:
+- click
+- xclim
+- h5netcdf