CLOUDP-80849: use temporary files for docker templating (#25)

Author: Rodrigo Valin

README.md

@@ -14,14 +14,15 @@ build Docker images.
 
 Sonar can be used as a Python module or as a standalone program. Sonar will look
 for an `inventory.yaml` file in your local directory that should contain a
-collection of images to build and stages for each one of those images.
+collection of images to build and stages for each one of those images. A
+different inventory file can be specified using `--inventory <file-path>`.
 
 Sonar comes with an inventory file to be able to build itself, and to run its
-unit tests. This [inventory.yaml](inventory.yaml) is:
+unit tests. This [simple.yaml](inventories/simple.yaml) is:
 
 ``` yaml
 vars:
-  # start a local Docker registry with:
+  # start a local registry with:
   # docker run -d -p 5000:5000 --restart=always --name registry registry:2
   registry: localhost:5000
 
@@ -49,32 +50,30 @@ images:
     task_type: tag_image
 
     source:
-      registry: $(inputs.params.registry)/sonar-tester-image
-      tag: $(inputs.params.version_id)
+      registry: $(stages['build-sonar-tester-image'].output[0].registry)
+      tag: $(stages['build-sonar-tester-image'].output[0].tag)
 
     destination:
-    - registry: $(inputs.params.registry)/sonar-tester-image-copy
-      tag: $(inputs.params.version_id)
-
+    - registry: $(inputs.params.registry)/sonar-tester-image
+      tag: latest
 ```
 
 To execute this inventory file, you can do:
 
 ```
-$ python sonar.py --image sonar-test-runner
+$ python sonar.py --image sonar-test-runner --inventory inventories/simple.yaml
 
 [build-sonar-tester-image/docker_build] stage-started build-sonar-tester-image: 1/2
 [build-sonar-tester-image/docker_build] docker-image-push: localhost:5000/sonar-tester-image:8945563b-248e-4c03-bb0a-6cc15cff1a6e
 [tag-image/tag_image] stage-started tag-image: 2/2
-[tag-image/tag_image] docker-image-push: localhost:5000/sonar-tester-image-copy:8945563b-248e-4c03-bb0a-6cc15cff1a6e
+[tag-image/tag_image] docker-image-push: localhost:5000/sonar-tester-image:latest
 ```
 
 At the end of this phase, you'll have a Docker image tagged as
-`localhost:5000/sonar-tester-image:8945563b-248e-4c03-bb0a-6cc15cff1a6e` that
-you will be able to run with:
+`localhost:5000/sonar-tester-image:latest` that you will be able to run with:
 
 ```
-$ docker run localhost:5000/sonar-tester-image:799170de-74a0-4310-a674-d704b83f2ed2
+$ docker run localhost:5000/sonar-tester-image:latest
 ============================= test session starts ==============================
 platform linux -- Python 3.9.4, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
 rootdir: /src

docker/Dockerfile.3.10rc

@@ -0,0 +1,5 @@
+{% extends "Dockerfile.template" %}
+
+{% set from_base = "python:3.10-rc-slim" %}
+
+# Sets the base version as 3.10 release candidate to try Sonar on latest Python!

docker/Dockerfile.template

@@ -0,0 +1,8 @@
+FROM {{ from_base|default("python:3.9-slim") }}
+
+COPY . /src
+
+RUN pip install -r /src/requirements.txt
+
+WORKDIR /src/
+ENTRYPOINT pytest

inventories/inventory-template.yaml

@@ -0,0 +1,49 @@
+## This is a more complex inventory file. It has a few features on it:
+##
+## 1. A dockerfile can be a Jinja2 template, like `docker/Dockerfile.template`
+## 2. This template dockerfile gets rendered into a concrete Dockerfile in a
+##    temp file on disk, using the $(functions.tempfile) function.
+## 3. The name of this tempfile is passed further and used by a subsequent
+##    stage using the `$(stages['stage-name'].outputs[0].dockerfile)`
+##
+## To run this inventory you have to:
+##
+## ./sonar.py --image sonar-test-runner --inventory inventories/inventory-template.yaml
+##
+
+vars:
+  # start a local registry with:
+  # docker run -d -p 5000:5000 --restart=always --name registry registry:2
+  registry: localhost:5000
+
+images:
+- name: sonar-test-runner
+
+  vars:
+    template_context: docker
+    context: .
+
+  # First stage builds a Docker image. The resulting image will be
+  # pushed to the registry in the `output` section.
+  stages:
+
+  - name: template-sonar
+    task_type: dockerfile_template
+    template_file_extension: 3.10rc  # Template will be `Dockerfile.3.10rc`
+
+    output:
+    # We will use $(functions.tempfile) to use a temporary file. The name of the
+    # temporary file will have to be accessed using
+    # `$(stages['stage-name']).outputs` afterwards.
+    - dockerfile: $(functions.tempfile)
+
+  - name: build-sonar-tester-image
+    task_type: docker_build
+
+    dockerfile: $(stages['template-sonar'].outputs[0].dockerfile)
+
+    output:
+    - registry: $(inputs.params.registry)/sonar-template-test
+      tag: $(inputs.params.version_id)
+    - registry: $(inputs.params.registry)/sonar-template-test
+      tag: latest

inventory.yaml renamed to inventories/simple.yaml

@@ -27,9 +27,9 @@ images:
     task_type: tag_image
 
     source:
-      registry: $(inputs.params.registry)/sonar-tester-image
-      tag: $(inputs.params.version_id)
+      registry: $(stages['build-sonar-tester-image'].outputs[0].registry)
+      tag: $(stages['build-sonar-tester-image'].outputs[0].tag)
 
     destination:
-    - registry: $(inputs.params.registry)/sonar-tester-image-copy
-      tag: $(inputs.params.version_id)
+    - registry: $(inputs.params.registry)/sonar-tester-image
+      tag: latest

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="sonar",
-    version="0.0.10",
+    version="0.0.11",
     description="Sonar Docker Building Tools",
     author="Rodrigo Valin",
     author_email="[email protected]",

sonar/sonar.py

@@ -11,10 +11,11 @@
 import subprocess
 import tempfile
 import uuid
+from collections import defaultdict
 from dataclasses import dataclass, field
 from pathlib import Path
 from shutil import copyfile
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union, Any
 from urllib.request import urlretrieve
 
 import boto3
@@ -79,6 +80,12 @@ class Context:
     # Generates a version_id to use if one is not present
     stored_version_id: str = str(uuid.uuid4())
 
+    # stage_outputs is a dictionary of dictionaries. First dict
+    # has a key corresponding to the name of the stage, the dict
+    # you get from it is key/value (str, Any) with the values
+    # stored by given stage.
+    stage_outputs: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
+
     # pylint: disable=C0103
     def I(self, string):
         """
@@ -101,6 +108,16 @@ def version_id(self):
         return os.environ.get("version_id", self.stored_version_id)
 
 
+def append_output_in_context(ctx: Context, stage_name: str, values: Dict) -> None:
+    """Stores a value as the output of the stage, so it can be consumed by future stages."""
+    if stage_name not in ctx.stage_outputs.keys():
+        # adds a new empty dictionary to this stage
+        # if there isn't one yet.
+        ctx.stage_outputs[stage_name] = list()
+
+    ctx.stage_outputs[stage_name].append(values)
+
+
 def find_inventory(inventory: Optional[str] = None):
     """
     Finds the inventory file, and return it as a yaml object.
@@ -124,14 +141,6 @@ def find_image(image_name: str, inventory: str):
     raise ValueError("Image {} not found".format(image_name))
 
 
-def find_variables_to_interpolate(string) -> List[str]:
-    """
-    Returns a list of variables in the string that need to be interpolated.
-    """
-    var_finder_re = r"\$\(inputs\.params\.(?P<var>\w+)\)"
-    return re.findall(var_finder_re, string, re.UNICODE)
-
-
 def find_variable_replacement(ctx: Context, variable: str, stage=None) -> str:
     """
     Returns the variable *value* for this varable.
@@ -187,6 +196,38 @@ def find_variable_replacements(
     return replacements
 
 
+def execute_interpolatable_function(name: str) -> str:
+    if name == "tempfile":
+        tmp = tempfile.mkstemp()
+        # mkstemp returns a tuple, with the second element of it being
+        # the absolute path to the file.
+        return tmp[1]
+
+    raise ValueError("Only supported function is 'tempfile'")
+
+
+def find_variables_to_interpolate_from_stage(string: str) -> List[Any]:
+    """Finds a $(stage['stage-name'].outputs[])"""
+    var_finder_re = r"\$\(stages\[\'(?P<stage_name>[\w-]+)\'\]\.outputs\[(?P<index>\d+)\]\.(?P<key>\w+)"
+
+    return re.findall(var_finder_re, string, re.UNICODE)
+
+
+def find_variables_to_interpolate(string) -> List[str]:
+    """
+    Returns a list of variables in the string that need to be interpolated.
+    """
+    var_finder_re = r"\$\(inputs\.params\.(?P<var>\w+)\)"
+    return re.findall(var_finder_re, string, re.UNICODE)
+
+
+def find_functions_to_interpolate(string: str) -> List[Any]:
+    """Find functions to be interpolated."""
+    var_finder_re = r"\$\(functions\.(?P<var>\w+)\)"
+
+    return re.findall(var_finder_re, string, re.UNICODE)
+
+
 def interpolate_vars(ctx: Context, string: str, stage=None) -> str:
     """
     For each variable to interpolate in string, finds its *value* and
@@ -200,8 +241,23 @@ def interpolate_vars(ctx: Context, string: str, stage=None) -> str:
             "$(inputs.params.{})".format(variable), replacements[variable]
         )
 
-    return string
+    variables = find_variables_to_interpolate_from_stage(string)
+    for stage, index, key in variables:
+        value = ctx.stage_outputs[stage][int(index)][key]
+        string = string.replace(
+            "$(stages['{}'].outputs[{}].{})".format(stage, index, key),
+            value
+        )
 
+    functions = find_functions_to_interpolate(string)
+    for name in functions:
+        value = execute_interpolatable_function(name)
+        string = string.replace(
+            "$(functions.{})".format(name),
+            value
+        )
+
+    return string
 
 def build_add_statement(ctx, block) -> str:
     """
@@ -334,6 +390,11 @@ def task_tag_image(ctx: Context):
             else:
                 raise
 
+        append_output_in_context(ctx, ctx.stage["name"], {
+            "registry": registry,
+            "tag": tag
+        })
+
 
 def get_rendering_params(ctx: Context) -> Dict[str, str]:
     """
@@ -532,6 +593,11 @@ def task_docker_build(ctx: Context):
             else:
                 raise
 
+        append_output_in_context(ctx, ctx.stage["name"], {
+            "registry": registry,
+            "tag": tag,
+        })
+
         if sign:
             clear_signing_environment(signing_key_name)
 
@@ -568,7 +634,13 @@ def task_dockerfile_template(ctx: Context):
     except KeyError:
         pass
 
-    dockerfile = run_dockerfile_template(ctx, template_context, ctx.stage.get("distro"))
+
+    template_file_extension = ctx.stage.get("template_file_extension")
+    if template_file_extension is None:
+        # Use distro as compatibility with pre 0.11
+        template_file_extension = ctx.stage.get("distro")
+
+    dockerfile = run_dockerfile_template(ctx, template_context, template_file_extension)
 
     for output in ctx.stage["output"]:
         if "dockerfile" in output:
@@ -577,6 +649,10 @@ def task_dockerfile_template(ctx: Context):
 
             echo(ctx, "dockerfile-save-location", output_dockerfile)
 
+            append_output_in_context(ctx, ctx.stage["name"], {
+                "dockerfile": output_dockerfile
+            })
+
 
 def find_skip_tags(params: Optional[Dict[str, str]] = None) -> List[str]:
     """Returns a list of tags passed in params that should be excluded from the build."""