st2bootstrap generic template

Makefile

@@ -1,25 +1,49 @@
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+SHELL := /bin/bash
+VIRTUALENV_DIR ?= virtualenv/st2packages
+.DEFAULT_GOAL := scriptsgen
+
+.PHONY: .create_venv
+.create_venv:
+	test -d "$(VIRTUALENV_DIR)" || python3 -m venv "$(VIRTUALENV_DIR)"
+
+.PHONY: .install_dependencies
+.install_dependencies:
+	"$(VIRTUALENV_DIR)/bin/pip3" install -r requirements.txt
+
+.PHONY: clean
+clean:
+	test -d "$(VIRTUALENV_DIR)" && rm -rf "$(VIRTUALENV_DIR)"
+
 .PHONY: scriptsgen
-scriptsgen:
+scriptsgen: .create_venv .install_dependencies
 	@echo
 	@echo "================== scripts gen ===================="
 	@echo
-	/usr/bin/env python3 tools/generate_final_installer_scripts.py
+	"$(VIRTUALENV_DIR)/bin/python3" tools/generate_install_script.py
+	# Remove comments to reduce script size by ~7k
+	for i in scripts/st2bootstrap-*.sh; \
+	do \
+		grep -Ev '^\s*#[^!]' "$$i" >"$$i".s && mv "$$i".s "$$i"; \
+	done
 
 .PHONY: .generated-files-check
 .generated-files-check:
 	# Verify that all the files which are automatically generated have indeed been re-generated and
 	# committed
 	@echo "==================== generated-files-check ===================="
 
-	# 1. Sample config - conf/st2.conf.sample
-	cp scripts/st2bootstrap-deb.sh /tmp/st2bootstrap-deb.sh.upstream
-	cp scripts/st2bootstrap-el8.sh /tmp/st2bootstrap-el8.sh.upstream
-	cp scripts/st2bootstrap-el9.sh /tmp/st2bootstrap-el9.sh.upstream
+	mkdir -p /tmp/scripts
+	for i in scripts/st2bootstrap-*.sh; \
+	do \
+		cp "$$i" "/tmp/$$i"; \
+	done
 
 	make scriptsgen
 
-	diff scripts/st2bootstrap-deb.sh /tmp/st2bootstrap-deb.sh.upstream || (echo "scripts/st2bootstrap-deb.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1)
-	diff scripts/st2bootstrap-el8.sh /tmp/st2bootstrap-el8.sh.upstream || (echo "scripts/st2bootstrap-el8.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1)
-	diff scripts/st2bootstrap-el9.sh /tmp/st2bootstrap-el9.sh.upstream || (echo "scripts/st2bootstrap-el9.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1)
+	for i in scripts/st2bootstrap-*.sh; \
+	do \
+		diff "$$i" "/tmp/$$i" || (echo "scripts/st2bootstrap-deb.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1); \
+	done
 
 	@echo "All automatically generated files are up to date."

readme-install-script.md

@@ -0,0 +1,60 @@
+# Installation script template
+
+## Overview
+
+The installation script is used to install StackStorm on to officially supported Linux distributions running on bare-metal or virtual machine systems.  The installation consists of applying system configuration and software installation required to run StackStorm and its dependencies on a single host.  A high-availability deployment is outside the scope of this script.  Consult the official documentation for instructions on how to deploy StackStorm in a high-availability configuration.
+
+## Components
+
+To maintain consistent logic and behaviour across multiple distributions and releases, there are two main phases to creating modular logical blocks of the script:
+
+1. Build time - Jinja template logic selectively includes, excludes or adapts code included in the output script.
+2. Run time - BASH script is written to use functions to apply system changes in logical blocks.
+
+## Jinja Template Logic
+
+Shell scripts are generated by running the make file.
+
+`make`
+
+The make file will setup a Python virtual environment, install jinja2 and read the set of Linux distributions to be generated from a JSON data file.
+
+The Jinja template engine will be given high level information to generate the script.  Below is the information to create the installation script for Ubuntu 20.04:
+```
+{
+    "id": "ubuntu",
+    "version_id": "20.04",
+    "script_filename": "st2bootstrap-focal.sh",
+    "pkg_mgr": "apt"
+}
+```
+The generation script and data file can be found in the `tools` directory.  The Jinja templates themselves are found in the `templates` directory.
+
+The main template is `st2bootstrap.jinja` which includes all other jinja templates.  Templates are written to group functional parts of the script into a single file to help maintainability and readability.
+
+- `funcs_display.jinja`: Functions related to formatting script output.
+- `funcs_mongodb.jinja`: Functions to configure and install mongodb on the system.
+- `funcs_package_manager.jinja`: Functions to install or query packages.
+- `funcs_rabbitmq.jinja`: Functions to configure and install rabbitmq on the system.
+- `funcs_redis.jinja`: Function to configure and install redis on the system.
+- `funcs_repo_manager.jinja`:  Functions to install and query package repositories.
+- `funcs_setup.jinja`: Functions to process script parameter input.
+- `funcs_st2chatops.jinja`: Functions to install ChatOps on the system.
+- `funcs_st2.jinja`: Functions to install StackStorm on the system.
+- `funcs_st2web.jinja`: Functions to install Web App on the system.
+- `funcs_system.jinja`: Functions for configuration and checks of installation pre-requisites.
+- `funcs_usage.jinja`: Functions for help output.
+
+In general, templating is used to conditionally include shell code based on distribution id, package manger or release id.
+Some special cases are
+  - RedHat type systems have a major.minor release (e.g. 9.1, 9.2 ...) but package repositories only make the distinction by major version.  To avoid the need to repeat entries in the data file for each minor version iteration only the major version is used.
+  - Debian type systems have versions (e.g. Ubuntu 20.04 or Debian 11) however the version code name is often used to select the repository.  This isn't required in the data file because the codename is sourced from `/etc/os-release`.
+
+Scripts are generated and written in the `scripts` directory.  After they're generated, shell comments are stripped to reduce the file size by approximately 15%.
+
+## Shell Logic
+
+As described above, shell logic is grouped by high level functions. These functions should avoid accessing global state or producing side effects, such as defining global variables from within the function calls.
+The only code that is executed immediately is the code located in the `st2bootstrap.jinja` template.  Shell variables are scoped to avoid name collisions or side effects between function calls.
+
+While only RedHat and Ubuntu type distributions are supported as of writing, the install script is designed to be as generic and flexible as possible to add other distributions or handle system evolutions in a clean, modular fashion.

requirements.txt

@@ -0,0 +1 @@
+Jinja2