Initial design pattern and features.

Author: Duncan Blythe

Diff for: .gitignore

@@ -0,0 +1,3 @@
+.idea
+.jd
+/checkpoints

Diff for: jobdeploy/__init__.py

@@ -0,0 +1,85 @@
+import click
+import json
+import os
+
+from controller import build_meta
+from resources import load_all_resources, load_resource
+
+
+@click.group()
+def cli():
+    ...
+
+
+class KeyValuePairs(click.ParamType):
+    """Convert to key value pairs"""
+    name = "key-value-pairs"
+
+    def convert(self, value, param, ctx):
+        """
+        Convert to key value pairs
+
+        :param value: value
+        :param param: parameter
+        :param ctx: context
+        """
+        if not value.strip():
+            return {}
+        try:
+            my_dict = dict([x.split('=') for x in value.split(',')])
+            for k, val in my_dict.items():
+                if val.isnumeric():
+                    my_dict[k] = eval(val)
+                elif val in {'true', 'True', 'false', 'False'}:
+                    my_dict[k] = val.lower == 'true'
+                elif '+' in val:
+                    val = val.split('+')
+                    val = [x for x in val if x]
+                    val = [eval(x) if x.isnumeric() else x for x in val]
+                    my_dict[k] = val
+            return my_dict
+        except TypeError:
+            self.fail(
+                "expected string for key-value-pairs() conversion, got "
+                f"{value!r} of type {type(value).__name__}",
+                param,
+                ctx,
+            )
+        except ValueError:
+            self.fail(f"{value!r} is not a valid key-value-pair", param, ctx)
+
+
+@cli.command()
+def ls():
+    print(json.dumps(load_all_resources(), indent=2))
+
+
+@cli.command()
+@click.argument('id')
+@click.option('--purge/--no-purge', default=False, help='purge resource')
+def rm(id, purge):
+    r = load_resource(id)
+    if 'stopped' not in r:
+        build_meta(r['template'], 'down', id=id)
+    if purge:
+        build_meta(r['template'], 'purge', id=id)
+
+    os.system(f'rm -rf .jd/{r["params"]["subdir"]}')
+
+
+@cli.command(help='build template')
+@click.argument('template')
+@click.argument('method')
+@click.option('--kwargs', default=None, help='key-value pairs to add to build',
+              type=KeyValuePairs())
+def build(template, method, kwargs):
+    print(kwargs)
+    if kwargs is None:
+        kwargs = {}
+    if template.endswith('.yaml'):
+        template = template.split('.yaml')[0]
+    build_meta(template, method, **kwargs)
+
+
+if __name__ == '__main__':
+    cli()

Diff for: jobdeploy/__main__.py

@@ -0,0 +1,60 @@
+import datetime
+import json
+import os
+import random
+
+from templates import call_template, load_template
+
+
+def random_id():
+    """ Random ID identifier."""
+    letters = list('ABDEFGHIJKLMNOPQRSTUVWXZYZ0123456789')
+    id_ = [random.choice(letters) for _ in range(8)]
+    return ''.join(id_)
+
+
+def build_meta(path, method, **params):
+    """
+    Call template at "path" with parameters.
+
+    :param path: Template .yaml path.
+    :param method: Name of build to run.
+    """
+    if method != 'up':
+        assert set(params.keys()) == {'id'}
+    else:
+        assert 'id' not in params
+        params['id'] = random_id()
+
+    prefix = path.replace('/', '-')
+    subdir = prefix + '-' + params['id']
+
+    if method != 'up':
+        with open(f'.jd/{subdir}/meta.json') as f:
+            meta = json.load(f)
+        params.update(meta['params'])
+
+    else:
+        params['subdir'] = subdir
+        os.system(f'mkdir -p .jd/{params["subdir"]}/tasks')
+        try:
+            with open('.jd/project.json') as f:
+                params.update(json.load(f))
+        except FileNotFoundError:
+            pass
+
+    template, binds = load_template(path)
+    if method == 'up':
+        params.update(binds)
+
+    call_template(template, method, **params)
+
+    if method == 'up':
+        meta = {'params': params, 'created': str(datetime.datetime.now()), 'template': path}
+        with open(f'.jd/{params["subdir"]}/meta.json', 'w') as f:
+            json.dump(meta, f)
+
+    if method == 'down':
+        meta['stopped'] = str(datetime.datetime.now())
+        with open(f'.jd/{params["subdir"]}/meta.json', 'w') as f:
+            json.dump(meta, f)

Diff for: jobdeploy/controller.py

@@ -0,0 +1,27 @@
+import json
+import os
+
+
+def load_all_resources():
+    """
+    Load all of the meta data of all deployments.
+    """
+    all_ = []
+    subdirs = [x for x in os.listdir('.jd') if x != 'project.json']
+    for subdir in subdirs:
+        with open(f'.jd/{subdir}/meta.json') as f:
+            meta = json.load(f)
+        all_.append(meta)
+    all_ = sorted(all_, key=lambda x: x['created'])
+    return all_
+
+
+def load_resource(id_):
+    """
+    Load meta data of id.
+
+    :param id_: ID identified of deployment.
+    """
+    subdir = [x for x in os.listdir('.jd') if x.endswith(id_)][0]
+    with open(f'.jd/{subdir}/meta.json') as f:
+        return json.load(f)

Diff for: jobdeploy/resources.py

@@ -0,0 +1,114 @@
+import json
+
+from jinja2 import Template, StrictUndefined
+import os
+import yaml
+
+
+def load_template(path):
+    """
+    Load deployment template.
+
+    :params path: Load template from path. If template has "parent" field, then combine with parent
+        template. Parameters named in binds are set in the parent by default.
+    """
+    with open(path + '.yaml') as f:
+        template = yaml.safe_load(f.read())
+    if not 'parent' in template:
+        return template
+    with open(template['parent']) as f:
+        parent = yaml.safe_load(f.read())
+    binds = template.get('binds', {})
+    parent['params'].extend([x for x in template.get('params', []) if x not in parent['params']])
+    for k in template.get('builds', {}):
+        parent['builds'][k] = template['builds'][k]
+    return parent, binds
+
+
+def create_value(value, other_values, params):
+    """
+    Format a value using template parameters.
+
+    :param value: Value to be formatted using Jinja2.
+    :param other_values: Other pre-built values to be used in the value with {{ values[...] }}.
+    :param params: Dictionary of parameters, referred to by {{ params[...] }}.
+    """
+    if isinstance(value, str):
+        return Template(value, undefined=StrictUndefined).render(params=params,
+                                                                 values=other_values)
+    elif isinstance(value, list):
+        return [create_value(x, other_values, params) for x in value]
+
+    elif isinstance(value, dict):
+        return {k: create_value(value[k], other_values, params) for k in value}
+
+    else:
+        raise NotImplementedError('only strings, and recursively lists and dicts supported')
+
+
+def create_values(values, **params):
+    """
+    Create values. Go through dictionary of values based on parameters dictionary.
+
+    :param values: Dictionary of values with Jinja2 variables in strings.
+    """
+    out = {}
+    for k in values:
+        out[k] = create_value(values[k], out, params)
+    return out
+
+
+def call_template(template, method, **params):
+    """
+    Call template with parameters.
+
+    :param template: Loaded template dictionary.
+    :param method: Name of build to run.
+    """
+
+    assert set(params.keys()) == set(template['params']), \
+        (f'missing keys: {set(template["params"]) - set(params.keys())}; '
+         f'unexpected keys: {set(params.keys()) - set(template["params"])}')
+
+    def build_method(method):
+        cf = template['builds'][method]
+        if method == 'up' and 'values' in template:
+            values = create_values(template, **params)
+            with open(params['subdir'] + '/values.json', 'w') as f:
+                json.dump(values, f)
+        else:
+            try:
+                with open(params['subdir'] + '/values.json') as f:
+                    values = json.load(f)
+            except FileNotFoundError:
+                values = {}
+        if cf['type'] != 'sequence':
+            print(f'building "{method}"')
+
+            content = Template(cf['content'], undefined=StrictUndefined).render(
+                params=params, values=values,
+            )
+            lines = content.split('\n')
+            len_ = max([len(x) for x in lines])
+            print(f'content:\n  ' + len_ * '-')
+            print('\n'.join(['  ' + x for x in lines]))
+            print(f'  ' + len_ * '-')
+            if cf['type'] == 'file':
+                path = f'.jd/{params["subdir"]}/tasks/{method}'
+                with open(path, 'w') as f:
+                    f.write(content)
+            if cf['type'] == 'script':
+                path = f'.jd/{params["subdir"]}/tasks/{method}'
+                with open(path, 'w') as f:
+                    f.write(content)
+                os.system(f'chmod +x {path}')
+                exit_code = os.system(f'./{path}')
+                if exit_code and exit_code not in cf.get('whitelist', []):
+                    raise Exception(f'script exited with non-zero exit code: {exit_code}.')
+            return
+
+        for m in cf['content']:
+            build_method(m)
+
+    build_method(method)
+

Diff for: jobdeploy/templates.py

@@ -0,0 +1,3 @@
+click
+pyyaml
+jinja2