fix baseCommand for docker runtime

Author: fmigneault

src/goldfinch/processes/indicator/hdd.cwl

@@ -2,20 +2,45 @@ $namespaces:
   s: https://schema.org/
 $schemas:
 - http://schema.org/version/9.0/schemaorg-current-http.rdf
-baseCommand: python /home/francis/dev/goldfinch/src/goldfinch/processes/indicator/hdd.py
+baseCommand: python -m goldfinch.processes.indicator.hdd
 class: CommandLineTool
 cwlVersion: v1.2
 hints:
   DockerRequirement:
     dockerPull: ghcr.io/bird-house/goldfinch:0.1.0
 id: hdd
 inputs:
+  chunks:
+    inputBinding:
+      position: 8
+      prefix: --chunks
+    type: string?
+  dask_maxmem:
+    inputBinding:
+      position: 7
+      prefix: --dask-maxmem
+    type: string?
+  dask_nthreads:
+    inputBinding:
+      position: 6
+      prefix: --dask-nthreads
+    type: None?
+  engine:
+    inputBinding:
+      position: 9
+      prefix: --engine
+    type: string?
+  help:
+    inputBinding:
+      position: 2
+      prefix: -h
+    type: boolean?
   indicator:
     inputBinding:
       position: 1
       prefix: --indicator
     type:
-    - symbols: !!python/tuple
+    - symbols:
       - HUMIDEX
       - HEAT_INDEX
       - TG
@@ -353,11 +378,6 @@ inputs:
       - cf.TXX
       - cf.VDTR
       type: enum
-  help:
-    inputBinding:
-      position: 2
-      prefix: -h
-    type: boolean?
   input:
     type:
     - 'null'
@@ -376,26 +396,6 @@ inputs:
       position: 5
       prefix: -v
     type: None?
-  dask_nthreads:
-    inputBinding:
-      position: 6
-      prefix: --dask-nthreads
-    type: None?
-  dask_maxmem:
-    inputBinding:
-      position: 7
-      prefix: --dask-maxmem
-    type: string?
-  chunks:
-    inputBinding:
-      position: 8
-      prefix: --chunks
-    type: string?
-  engine:
-    inputBinding:
-      position: 9
-      prefix: --engine
-    type: string?
 outputs:
   results:
     outputBinding:

src/goldfinch/processes/indicator/hdd.py

@@ -1,6 +1,5 @@
 import click
 import warnings
-import xclim.cli
 import xclim.core.indicator
 
 """
@@ -21,31 +20,10 @@
 The CLI's output is a user input. We'll probably want to change this to a default output path.
 """
 
-# WARNING: this does not work,
-#  the click2cwl tool cannot be aware of subcommands, since they are not instantiated until invocation runtime
-#  however, the tool purposely avoids command invocation to avoid triggering the operations
-class Cli(click.Group):
-    """Main cli class."""
-
-    def list_commands(self, ctx):
-        """Return the available commands (other than the indicators)."""
-        return (
-            "heating_degree_days",
-        )
-
-    def get_command(self, ctx, cmd_name):
-        """Return the requested command."""
-        if cmd_name in self.list_commands(ctx):
-            command = xclim.cli._create_command(cmd_name)
-        return command
-
 
 @click.command(
-    #cls=Cli,
-    #chain=True,
     help="Command line tool to compute indices on netCDF datasets. Indicators are referred to by their "
     "(case-insensitive) identifier, as in xclim.core.indicator.registry.",
-    #invoke_without_command=True,
 )
 @click.option(  # WARNING: click.argument(help='...') is not supported, but click2cwl requires an 'help'
     "--indicator",
@@ -102,7 +80,7 @@ def cli(ctx, **kwargs):
         kwargs["input"] = kwargs["input"][0]
 
     if kwargs["dask_nthreads"] is not None:
-        if not distributed:
+        if not distributed:  # FIXME: undefined
             raise click.BadOptionUsage(
                 "dask_nthreads",
                 "Dask's distributed scheduler is not installed, only the "
@@ -116,7 +94,7 @@ def cli(ctx, **kwargs):
                 ctx,
             )
 
-        client = Client(
+        client = Client(  # FIXME: undefined
             n_workers=1,
             threads_per_worker=kwargs["dask_nthreads"],
             memory_limit=kwargs["dask_maxmem"],
@@ -136,8 +114,6 @@ def cli(ctx, **kwargs):
     }
     ctx.obj = kwargs
 
-#cli.result_callback()(click.pass_context(xclim.cli.write_file))
-
 
 if __name__ == "__main__":
     cli()  # pylint: disable=no-value-for-parameter

src/goldfinch/processes/subset/poly_subset.cwl

@@ -2,14 +2,29 @@ $namespaces:
   s: https://schema.org/
 $schemas:
 - http://schema.org/version/9.0/schemaorg-current-http.rdf
-baseCommand: python /home/francis/dev/goldfinch/src/goldfinch/processes/subset/poly_subset.py
+baseCommand: python -m goldfinch.processes.subset.poly_subset
 class: CommandLineTool
 cwlVersion: v1.2
 hints:
   DockerRequirement:
     dockerPull: ghcr.io/bird-house/goldfinch:0.1.0
 id: poly_subset
 inputs:
+  buffer:
+    inputBinding:
+      position: 5
+      prefix: -b
+    type: None?
+  end:
+    inputBinding:
+      position: 7
+      prefix: -e
+    type: string?
+  first_level:
+    inputBinding:
+      position: 8
+      prefix: -f
+    type: string?
   help:
     inputBinding:
       position: 1
@@ -20,41 +35,26 @@ inputs:
       position: 2
       prefix: -i
     type: string?
+  last_level:
+    inputBinding:
+      position: 9
+      prefix: -l
+    type: string?
   output:
     inputBinding:
       position: 3
       prefix: -o
-    type: Directory?
+    type: File?
   poly:
     inputBinding:
       position: 4
       prefix: -p
-    type: Directory?
-  buffer:
-    inputBinding:
-      position: 5
-      prefix: -b
-    type: None?
+    type: File?
   start:
     inputBinding:
       position: 6
       prefix: -s
     type: string?
-  end:
-    inputBinding:
-      position: 7
-      prefix: -e
-    type: string?
-  first_level:
-    inputBinding:
-      position: 8
-      prefix: -f
-    type: string?
-  last_level:
-    inputBinding:
-      position: 9
-      prefix: -l
-    type: string?
   verbose:
     inputBinding:
       position: 10

src/goldfinch/utils/click2cwl_cli.py

@@ -67,11 +67,40 @@ def yaml_dump(data: dict[str, Any], *args: Any, **kwargs: Any) -> str:
     """
     Custom YAML dump function to ensure consistent formatting.
     """
-    represent_dict_order = lambda self, _data: self.represent_mapping("tag:yaml.org,2002:map", _data.items())
+    represent_dict_order = lambda self, _data: self.represent_mapping("tag:yaml.org,2002:map", dict(_data))
     yaml.add_representer(OrderedDict, represent_dict_order)
     return yaml.dump(data, *args, **kwargs)
 
 
+def resolve_cli_command(
+    process: str | None = None,
+    docker: str | None = None,
+    docker_module: str | None = None,
+    **_,
+) -> str:
+    """
+    Resolve the command line interface program call.
+
+    If running inside docker, patch the command with **installed** module or the overridden script/module reference.
+    Otherwise, employ the absolute path of the python script directly for local execution (not within docker).
+    """
+    if not docker:
+        return f"python {process}"
+
+    if docker_module and os.path.sep in str(docker_module):
+        return f"python {process}"
+
+    # allow directly invoking other modules if they are installed, or resolve current package
+    parent_mod_name = "goldfinch"
+    parent_module = f"{parent_mod_name}{os.path.sep}"
+    docker_module = docker_module or process
+    if f"{os.path.sep}{parent_module}" in docker_module or docker_module.startswith(parent_module):
+        docker_module = docker_module.rstrip(".py").rsplit(parent_module, 1)[-1].replace(os.path.sep, ".")
+        docker_module = f"{parent_mod_name}.{docker_module}"
+
+    return f"python -m {docker_module}"
+
+
 @click.command(
     "click2cwl",
     context_settings={"allow_extra_args": True},  # pass down job parameters to context
@@ -184,7 +213,7 @@ def main(ctx: click.Context, **kwargs: str) -> None:
     """
     # resolve the module containing the Click command
     cli_file = kwargs["process"]
-    cli_path = os.path.abspath(cli_file)
+    cli_path = kwargs["process"] = os.path.abspath(cli_file)
     cli_spec = importlib.util.spec_from_file_location("click2cwl.program", cli_path)
     sys.modules["click2cwl.program"] = cli_spec  # type: ignore
     cli_mod = importlib.util.module_from_spec(cli_spec)
@@ -204,7 +233,7 @@ def main(ctx: click.Context, **kwargs: str) -> None:
             f"Use the --command option to specify a specific one to convert or make sure the module defines one."
         )
     cli_name, cli_cmd = click_functions[0]
-    cli_prog_call = f"python {cli_file}"
+    cli_prog_call = resolve_cli_command(**kwargs)
 
     # add context arguments to generate the CWL contents
     cli_ctx = cli_cmd.context_class(info_name=cli_prog_call, command=cli_cmd)

tests/utils/test_click2clw_cli.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import pytest
+
+from goldfinch.utils.click2cwl_cli import resolve_cli_command
+
+
+@pytest.mark.parametrize(
+    ["expected_command", "process", "docker", "docker_module"],
+    [
+        (
+            "python -m xclim.cli",
+            "xclim.cli",
+            "bird-house/goldfinch:latest",
+            None,
+        ),
+        (
+            "python -m goldfinch.processes.subset.poly_subset",
+            "goldfinch.processes.subset.poly_subset",
+            "bird-house/goldfinch:latest",
+            None,
+        ),
+        (
+            "python -m goldfinch.processes.subset.poly_subset",
+            "goldfinch/processes/subset/poly_subset.py",
+            "bird-house/goldfinch:latest",
+            None,
+        ),
+        (
+            "python -m goldfinch.processes.subset.poly_subset",
+            "src/goldfinch/processes/subset/poly_subset.py",
+            "bird-house/goldfinch:latest",
+            None,
+        ),
+        (
+            "python -m goldfinch.processes.subset.poly_subset",
+            "/home/user/goldfinch/processes/subset/poly_subset.py",
+            "bird-house/goldfinch:latest",
+            None,
+        ),
+        (
+            "python -m goldfinch.processes.subset.poly_subset",
+            "poly_subset.py",
+            "bird-house/goldfinch:latest",
+            "goldfinch.processes.subset.poly_subset",
+        ),
+        (
+            "python -m xclim.cli",
+            "goldfinch.processes.indicator.hdd",
+            "bird-house/goldfinch:latest",
+            "xclim.cli",
+        ),
+        (
+            "python /home/user/goldfinch/processes/subset/poly_subset.py",
+            "/home/user/goldfinch/processes/subset/poly_subset.py",
+            None,
+            None,
+        ),
+    ]
+)
+def test_resolve_cli_command(expected_command, process, docker, docker_module):
+    result = resolve_cli_command(process, docker, docker_module)
+    assert result == expected_command