diff --git a/Makefile b/Makefile index 82b82737..c0e757f5 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,31 @@ +ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +SHELL := /bin/bash +VIRTUALENV_DIR ?= virtualenv/st2packages +.DEFAULT_GOAL := scriptsgen + +.PHONY: .create_venv +.create_venv: + test -d "$(VIRTUALENV_DIR)" || python3 -m venv "$(VIRTUALENV_DIR)" + +.PHONY: .install_dependencies +.install_dependencies: + "$(VIRTUALENV_DIR)/bin/pip3" install -r requirements.txt + +.PHONY: clean +clean: + test -d "$(VIRTUALENV_DIR)" && rm -rf "$(VIRTUALENV_DIR)" + .PHONY: scriptsgen -scriptsgen: +scriptsgen: .create_venv .install_dependencies @echo @echo "================== scripts gen ====================" @echo - /usr/bin/env python3 tools/generate_final_installer_scripts.py + "$(VIRTUALENV_DIR)/bin/python3" tools/generate_install_script.py + # Remove comments to reduce script size by ~7k + for i in scripts/st2bootstrap-*.sh; \ + do \ + grep -Ev '^\s*#[^!]' "$$i" >"$$i".s && mv "$$i".s "$$i"; \ + done .PHONY: .generated-files-check .generated-files-check: @@ -11,15 +33,17 @@ scriptsgen: # committed @echo "==================== generated-files-check ====================" - # 1. Sample config - conf/st2.conf.sample - cp scripts/st2bootstrap-deb.sh /tmp/st2bootstrap-deb.sh.upstream - cp scripts/st2bootstrap-el8.sh /tmp/st2bootstrap-el8.sh.upstream - cp scripts/st2bootstrap-el9.sh /tmp/st2bootstrap-el9.sh.upstream + mkdir -p /tmp/scripts + for i in scripts/st2bootstrap-*.sh; \ + do \ + cp "$$i" "/tmp/$$i"; \ + done make scriptsgen - diff scripts/st2bootstrap-deb.sh /tmp/st2bootstrap-deb.sh.upstream || (echo "scripts/st2bootstrap-deb.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1) - diff scripts/st2bootstrap-el8.sh /tmp/st2bootstrap-el8.sh.upstream || (echo "scripts/st2bootstrap-el8.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1) - diff scripts/st2bootstrap-el9.sh /tmp/st2bootstrap-el9.sh.upstream || (echo "scripts/st2bootstrap-el9.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1) + for i in scripts/st2bootstrap-*.sh; \ + do \ + diff "$$i" "/tmp/$$i" || (echo "scripts/st2bootstrap-deb.sh hasn't been re-generated and committed. Please run \"make scriptsgen\" and include and commit the generated file." && exit 1); \ + done @echo "All automatically generated files are up to date." diff --git a/readme-install-script.md b/readme-install-script.md new file mode 100644 index 00000000..90c363e7 --- /dev/null +++ b/readme-install-script.md @@ -0,0 +1,60 @@ +# Installation script template + +## Overview + +The installation script is used to install StackStorm on to officially supported Linux distributions running on bare-metal or virtual machine systems. The installation consists of applying system configuration and software installation required to run StackStorm and its dependencies on a single host. A high-availability deployment is outside the scope of this script. Consult the official documentation for instructions on how to deploy StackStorm in a high-availability configuration. + +## Components + +To maintain consistent logic and behaviour across multiple distributions and releases, there are two main phases to creating modular logical blocks of the script: + +1. Build time - Jinja template logic selectively includes, excludes or adapts code included in the output script. +2. Run time - BASH script is written to use functions to apply system changes in logical blocks. + +## Jinja Template Logic + +Shell scripts are generated by running the make file. + +`make` + +The make file will setup a Python virtual environment, install jinja2 and read the set of Linux distributions to be generated from a JSON data file. + +The Jinja template engine will be given high level information to generate the script. Below is the information to create the installation script for Ubuntu 20.04: +``` +{ + "id": "ubuntu", + "version_id": "20.04", + "script_filename": "st2bootstrap-focal.sh", + "pkg_mgr": "apt" +} +``` +The generation script and data file can be found in the `tools` directory. The Jinja templates themselves are found in the `templates` directory. + +The main template is `st2bootstrap.jinja` which includes all other jinja templates. Templates are written to group functional parts of the script into a single file to help maintainability and readability. + +- `funcs_display.jinja`: Functions related to formatting script output. +- `funcs_mongodb.jinja`: Functions to configure and install mongodb on the system. +- `funcs_package_manager.jinja`: Functions to install or query packages. +- `funcs_rabbitmq.jinja`: Functions to configure and install rabbitmq on the system. +- `funcs_redis.jinja`: Function to configure and install redis on the system. +- `funcs_repo_manager.jinja`: Functions to install and query package repositories. +- `funcs_setup.jinja`: Functions to process script parameter input. +- `funcs_st2chatops.jinja`: Functions to install ChatOps on the system. +- `funcs_st2.jinja`: Functions to install StackStorm on the system. +- `funcs_st2web.jinja`: Functions to install Web App on the system. +- `funcs_system.jinja`: Functions for configuration and checks of installation pre-requisites. +- `funcs_usage.jinja`: Functions for help output. + +In general, templating is used to conditionally include shell code based on distribution id, package manger or release id. +Some special cases are + - RedHat type systems have a major.minor release (e.g. 9.1, 9.2 ...) but package repositories only make the distinction by major version. To avoid the need to repeat entries in the data file for each minor version iteration only the major version is used. + - Debian type systems have versions (e.g. Ubuntu 20.04 or Debian 11) however the version code name is often used to select the repository. This isn't required in the data file because the codename is sourced from `/etc/os-release`. + +Scripts are generated and written in the `scripts` directory. After they're generated, shell comments are stripped to reduce the file size by approximately 15%. + +## Shell Logic + +As described above, shell logic is grouped by high level functions. These functions should avoid accessing global state or producing side effects, such as defining global variables from within the function calls. +The only code that is executed immediately is the code located in the `st2bootstrap.jinja` template. Shell variables are scoped to avoid name collisions or side effects between function calls. + +While only RedHat and Ubuntu type distributions are supported as of writing, the install script is designed to be as generic and flexible as possible to add other distributions or handle system evolutions in a clean, modular fashion. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8ce973e9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Jinja2 diff --git a/scripts/includes/common.sh b/scripts/includes/common.sh deleted file mode 100644 index 8094f894..00000000 --- a/scripts/includes/common.sh +++ /dev/null @@ -1,307 +0,0 @@ -function configure_proxy() { - # Allow bypassing 'proxy' env vars via sudo - local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' - if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" - fi - - # Configure proxy env vars for 'st2api', 'st2actionrunner' and 'st2chatops' system configs - # See: https://docs.stackstorm.com/packs.html#installing-packs-from-behind-a-proxy - local service_config_path=$(hash apt-get >/dev/null 2>&1 && echo '/etc/default' || echo '/etc/sysconfig') - for service in st2api st2actionrunner st2chatops; do - service_config="${service_config_path}/${service}" - # create file if doesn't exist yet - sudo test -e ${service_config} || sudo touch ${service_config} - for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do - # delete line from file if specific proxy env var is unset - if sudo test -z "${!env_var:-}"; then - sudo sed -i "/^${env_var}=/d" ${service_config} - # add proxy env var if it doesn't exist yet - elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then - sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" - # modify existing proxy env var value - elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then - sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} - fi - done - done -} - -function get_package_url() { - # Retrieve direct package URL for the provided dev build, subtype and package name regex. - DEV_BUILD=$1 # Repo name and build number - / (e.g. st2/5646) - DISTRO=$2 # Distro name (e.g. focal,jammy,el8,el9) - - PACKAGE_NAME_REGEX=$3 - - PACKAGES_METADATA=$(curl -sSL -q https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts) - - if [ -z "${PACKAGES_METADATA}" ]; then - echo "Failed to retrieve packages metadata from https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" 1>&2 - return 2 - fi - - PACKAGES_URLS="$(echo ${PACKAGES_METADATA} | jq -r '.[].url')" - PACKAGE_URL=$(echo "${PACKAGES_URLS}" | egrep "${DISTRO}/${PACKAGE_NAME_REGEX}") - - if [ -z "${PACKAGE_URL}" ]; then - echo "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" 1>&2 - echo "Circle CI response: ${PACKAGES_METADATA}" 1>&2 - return 2 - fi - - echo ${PACKAGE_URL} -} - - -function port_status() { - # If the specified tcp4 port is bound, then return the "port pid/procname", - # else if a pipe command fails, return "Unbound", - # else return "". - # - # Please note that all return values end with a newline. - # - # Use netstat and awk to get a list of all the tcp4 sockets that are in the LISTEN state, - # matching the specified port. - # - # The `netstat -tunlp --inet` command is assumed to output data in the following format: - # Active Internet connections (only servers) - # Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name - # tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7506/httpd - # - # The awk command prints the 4th and 7th columns of any line matching both the following criteria: - # 1) The 4th column contains the port passed to port_status() (i.e., $1) - # 2) The 6th column contains "LISTEN" - # - # Sample output: - # 0.0.0.0:25000 7506/sshd - ret=$(sudo netstat -tunlp --inet | awk -v port=":$1$" '$4 ~ port && $6 ~ /LISTEN/ { print $4 " " $7 }' || echo 'Unbound'); - echo "$ret"; -} - - -check_st2_host_dependencies() { - # CHECK 1: Determine which, if any, of the required ports are used by an existing process. - - # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) - # and st2 (9100-9102). - - declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") - declare -a used=() - - for i in "${ports[@]}" - do - rv=$(port_status $i | sed 's/.*-$\|.*systemd\|.*beam.smp.*\|.*epmd\|.*st2.*\|.*nginx.*\|.*python.*\|.*postmaster.*\|.*mongod\|.*init//') - if [ "$rv" != "Unbound" ] && [ "$rv" != "" ]; then - used+=("$rv") - fi - done - - # If any used ports were found, display helpful message and exit - if [ ${#used[@]} -gt 0 ]; then - printf "\nNot all required TCP ports are available. ST2 and related services will fail to start.\n\n" - echo "The following ports are in use by the specified pid/process and need to be stopped:" - for port_pid_process in "${used[@]}" - do - echo " $port_pid_process" - done - echo "" - exit 1 - fi - - # CHECK 2: Ensure there is enough space at /var/lib/mongodb - VAR_SPACE=`df -Pk /var/lib | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{print $4}'` - if [ ${VAR_SPACE} -lt 358400 ]; then - echo "" - echo "MongoDB requires at least 350MB free in /var/lib/mongodb" - echo "There is not enough space for MongoDB. It will fail to start." - echo "Please, add some space to /var or clean it up." - exit 1 - fi -} - - -generate_random_passwords() { - # Generate random password used for MongoDB user authentication - ST2_MONGODB_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') - # Generate random password used for RabbitMQ user authentication - ST2_RABBITMQ_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') -} - - -configure_st2_user () { - # Create an SSH system user (default `stanley` user may be already created) - if (! id stanley 2>/dev/null); then - sudo useradd stanley - fi - - SYSTEM_HOME=$(echo ~stanley) - - if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then - sudo mkdir ${SYSTEM_HOME}/.ssh - sudo chmod 700 ${SYSTEM_HOME}/.ssh - fi - - # Generate ssh keys on StackStorm box and copy over public key into remote box. - # NOTE: If the file already exists and is non-empty, then assume the key does not need - # to be generated again. - if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then - # added PEM to enforce PEM ssh key type in EL8 to maintain consistency - sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM - fi - - if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; - then - # Authorize key-base access - sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" - fi - - sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys - sudo chmod 0700 ${SYSTEM_HOME}/.ssh - sudo chown -R stanley:stanley ${SYSTEM_HOME} - - # Enable passwordless sudo - local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" - if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" - fi - - sudo chmod 0440 /etc/sudoers.d/st2 - - # Disable requiretty for all users - sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers -} - - -configure_st2_cli_config() { - # Configure CLI config (write credentials for the root user and user which ran the script) - ROOT_USER="root" - CURRENT_USER=$(whoami) - - ROOT_HOME=$(eval echo ~${ROOT_USER}) - : "${HOME:=$(eval echo ~${CURRENT_USER})}" - - ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" - ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" - - CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" - CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" - - if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then - sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} - fi - - sudo sh -c "cat < ${ROOT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" - - # Write config for root user - if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then - return - fi - - # Write config for current user (in case current user != root) - if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then - sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} - fi - - sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" - - # Fix the permissions - sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} -} - - -generate_symmetric_crypto_key_for_datastore() { - DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" - DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" - - sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - - # If the file ${DATASTORE_ENCRYPTION_KEY_PATH} exists and is not empty, then do not generate - # a new key. st2-generate-symmetric-crypto-key fails if the key file already exists. - if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then - sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} - fi - - # Make sure only st2 user can read the file - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEY_PATH} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEY_PATH} - - # set path to the key file in the config - sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} - - # NOTE: We need to restart all the affected services so they pick the key and load it in memory - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2sensorcontainer - sudo st2ctl restart-component st2workflowengine - sudo st2ctl restart-component st2actionrunner -} - - -verify_st2() { - st2 --version - st2 -h - - st2 auth $USERNAME -p $PASSWORD - # A shortcut to authenticate and export the token - export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) - - # List the actions from a 'core' pack - st2 action list --pack=core - - # Run a local shell command - st2 run core.local -- date -R - - # See the execution results - st2 execution list - - # Fire a remote comand via SSH (Requires passwordless SSH) - st2 run core.remote hosts='127.0.0.1' -- uname -a - - # Install a pack - st2 pack install st2 -} - - -ok_message() { - echo "" - echo "" - echo "███████╗████████╗██████╗ ██████╗ ██╗ ██╗"; - echo "██╔════╝╚══██╔══╝╚════██╗ ██╔═══██╗██║ ██╔╝"; - echo "███████╗ ██║ █████╔╝ ██║ ██║█████╔╝ "; - echo "╚════██║ ██║ ██╔═══╝ ██║ ██║██╔═██╗ "; - echo "███████║ ██║ ███████╗ ╚██████╔╝██║ ██╗"; - echo "╚══════╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝"; - echo "" - echo " st2 is installed and ready to use." - echo "" - echo "Head to https://YOUR_HOST_IP/ to access the WebUI" - echo "" - echo "Don't forget to dive into our documentation! Here are some resources" - echo "for you:" - echo "" - echo "* Documentation - https://docs.stackstorm.com" - echo "* Pack Exchange - https://exchange.stackstorm.org/" - echo "" - echo "Thanks for installing StackStorm! Come visit us in our Slack Channel" - echo "and tell us how it's going. We'd love to hear from you!" - echo "http://stackstorm.com/community-signup" -} - - -fail() { - echo "############### ERROR ###############" - echo "# Failed on $STEP #" - echo "#####################################" - exit 2 -} diff --git a/scripts/includes/rhel.sh b/scripts/includes/rhel.sh deleted file mode 100644 index 3d86a888..00000000 --- a/scripts/includes/rhel.sh +++ /dev/null @@ -1,50 +0,0 @@ -install_yum_utils() { - # We need repoquery tool to get package_name-package_ver-package_rev in RPM based distros - # if we don't want to construct this string manually using yum info --show-duplicates and - # doing a bunch of sed awk magic. Problem is this is not installed by default on all images. - sudo yum install -y yum-utils -} - - -get_full_pkg_versions() { - if [ "$VERSION" != '' ]; - then - local RHMAJVER=`cat /etc/redhat-release | sed 's/[^0-9.]*\([0-9.]\).*/\1/'` - local YES_FLAG="" - if [ "$RHMAJVER" -ge "8" ]; then - # RHEL 8 and newer, you need "-y" flag to avoid being prompted to confirm "yes" - local YES_FLAG="-y" - fi - - local ST2_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2 | grep -F st2-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2_VER" ]; then - echo "Could not find requested version of st2!!!" - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2 - exit 3 - fi - ST2_PKG=${ST2_VER} - - local ST2WEB_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2web | grep -F st2web-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2WEB_VER" ]; then - echo "Could not find requested version of st2web." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2web - exit 3 - fi - ST2WEB_PKG=${ST2WEB_VER} - - local ST2CHATOPS_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops | grep -F st2chatops-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2CHATOPS_VER" ]; then - echo "Could not find requested version of st2chatops." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops - exit 3 - fi - ST2CHATOPS_PKG=${ST2CHATOPS_VER} - - echo "##########################################################" - echo "#### Following versions of packages will be installed ####" - echo "${ST2_PKG}" - echo "${ST2WEB_PKG}" - echo "${ST2CHATOPS_PKG}" - echo "##########################################################" - fi -} diff --git a/scripts/st2bootstrap-deb.sh b/scripts/st2bootstrap-deb.sh index a399f353..8e4e76b2 100644 --- a/scripts/st2bootstrap-deb.sh +++ b/scripts/st2bootstrap-deb.sh @@ -1,528 +1,913 @@ #!/usr/bin/env bash -# NOTE: This file is automatically generated by the tools/generate_final_installer_scripts.py -# script using the template file and common include files in scripts/includes/*.sh. # -# DO NOT EDIT MANUALLY. # -# Please edit corresponding template file and include files. -set -eu +set -e -u +x HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} VERSION='' RELEASE='stable' REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' -ST2WEB_PKG_VERSION='' -ST2CHATOPS_PKG_VERSION='' DEV_BUILD='' USERNAME='' PASSWORD='' -U16_ADD_INSECURE_PY3_PPA=0 -SUBTYPE=`lsb_release -cs` +ST2_PKG='st2' +ST2WEB_PKG='st2web' +ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 -if [[ "$SUBTYPE" != 'focal' ]]; then - echo "Unsupported ubuntu codename ${SUBTYPE}. Please use Ubuntu 20.04 (focal) as base system!" - exit 2 -fi +declare -A INSTALL_TYPE=() -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - # Provide flag to enable installing Python3 from 3rd party insecure PPA for Ubuntu Xenial - # TODO: Remove once Ubuntu Xenial is dropped - --u16-add-insecure-py3-ppa) - U16_ADD_INSECURE_PY3_PPA=1 - shift - ;; - *) - # unknown option - ;; - esac - done +source <(sed 's/^/OS_/g' /etc/os-release) + +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + + echo "${OUTPUT}" +} +function cecho() +{ + if [[ "$1" == "-n" ]]; then + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} +setup_install_parameters() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + if [[ -n "$DEV_BUILD" ]]; then + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - if [[ "$VERSION" != '' ]]; then +setup_check_version() +{ + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing StackStorm $RELEASE $VERSION " - echo "########################################################" - - if [ "$REPO_TYPE" == "staging" ]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [ "$DEV_BUILD" != '' ]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [ "${PASSWORD}" = '' ]; then - echo "Password cannot be empty." - exit 1 + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi fi - fi - -} - - -function configure_proxy() { - # Allow bypassing 'proxy' env vars via sudo - local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' - if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" - fi - - # Configure proxy env vars for 'st2api', 'st2actionrunner' and 'st2chatops' system configs - # See: https://docs.stackstorm.com/packs.html#installing-packs-from-behind-a-proxy - local service_config_path=$(hash apt-get >/dev/null 2>&1 && echo '/etc/default' || echo '/etc/sysconfig') - for service in st2api st2actionrunner st2chatops; do - service_config="${service_config_path}/${service}" - # create file if doesn't exist yet - sudo test -e ${service_config} || sudo touch ${service_config} - for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do - # delete line from file if specific proxy env var is unset - if sudo test -z "${!env_var:-}"; then - sudo sed -i "/^${env_var}=/d" ${service_config} - # add proxy env var if it doesn't exist yet - elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then - sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" - # modify existing proxy env var value - elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then - sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} - fi - done - done + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" } -function get_package_url() { - # Retrieve direct package URL for the provided dev build, subtype and package name regex. - DEV_BUILD=$1 # Repo name and build number - / (e.g. st2/5646) - DISTRO=$2 # Distro name (e.g. focal,jammy,el8,el9) - PACKAGE_NAME_REGEX=$3 +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} +pkg_install() +{ + + export DEBIAN_FRONTEND=noninteractive + + sudo apt -y install $@ +} - PACKAGES_METADATA=$(curl -sSL -q https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts) +pkg_meta_update() +{ + + sudo apt update -y + +} + +pkg_is_installed() +{ + PKG="$1" + + dpkg -l "$PKG" | grep -qE "^ii.*${PKG}" + +} + + + +repo_definition() +{ + REPO_PATH="/etc/apt/sources.list.d" + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + + REPO_NAME="$1" + REPO_URL="$2" + REPO_SUITES="$3" + REPO_COMPONENT="$4" + KEY_NAME="$5" + KEY_URL="$6" + + repo_add_gpg_key "$KEY_NAME" "$KEY_URL" + sudo cat <"${REPO_PATH}/${REPO_NAME}.sources" +Types: deb +URIs: ${REPO_URL} +Suites: ${REPO_SUITES} +Components: ${REPO_COMPONENT} +Architectures: $(dpkg --print-architecture) +Signed-By: ${GPG_KEY_PATH}/${KEY_NAME}.gpg +EOF +} - if [ -z "${PACKAGES_METADATA}" ]; then - echo "Failed to retrieve packages metadata from https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" 1>&2 - return 2 - fi - PACKAGES_URLS="$(echo ${PACKAGES_METADATA} | jq -r '.[].url')" - PACKAGE_URL=$(echo "${PACKAGES_URLS}" | egrep "${DISTRO}/${PACKAGE_NAME_REGEX}") +repo_add_gpg_key() +{ + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + KEY_NAME="$1" + KEY_URL="$2" + + curl -1sLf "$KEY_URL" | sudo gpg --dearmor -o "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" + sudo mv "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" "${GPG_KEY_PATH}/${KEY_NAME}.gpg" +} - if [ -z "${PACKAGE_URL}" ]; then - echo "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" 1>&2 - echo "Circle CI response: ${PACKAGES_METADATA}" 1>&2 - return 2 - fi - echo ${PACKAGE_URL} +pkg_get_latest_version() +{ + local PKG="$1" + local VERSION="$2" + apt-cache show "$PKG" | awk '/Version:/{print $2}' | grep "^${VERSION//./\\.}" | sort --version-sort | tail -n 1 } -function port_status() { - # If the specified tcp4 port is bound, then return the "port pid/procname", - # else if a pipe command fails, return "Unbound", - # else return "". - # - # Please note that all return values end with a newline. - # - # Use netstat and awk to get a list of all the tcp4 sockets that are in the LISTEN state, - # matching the specified port. - # - # The `netstat -tunlp --inet` command is assumed to output data in the following format: - # Active Internet connections (only servers) - # Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name - # tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7506/httpd - # - # The awk command prints the 4th and 7th columns of any line matching both the following criteria: - # 1) The 4th column contains the port passed to port_status() (i.e., $1) - # 2) The 6th column contains "LISTEN" - # - # Sample output: - # 0.0.0.0:25000 7506/sshd - ret=$(sudo netstat -tunlp --inet | awk -v port=":$1$" '$4 ~ port && $6 ~ /LISTEN/ { print $4 " " $7 }' || echo 'Unbound'); - echo "$ret"; +pkg_get_versions() +{ + apt-cache show "$1" } -check_st2_host_dependencies() { - # CHECK 1: Determine which, if any, of the required ports are used by an existing process. +repo_clean_meta() +{ + true +} - # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) - # and st2 (9100-9102). - declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") - declare -a used=() +repo_pkg_availability() { + local PKG="$1" + local VERSION="$2" - for i in "${ports[@]}" - do - rv=$(port_status $i | sed 's/.*-$\|.*systemd\|.*beam.smp.*\|.*epmd\|.*st2.*\|.*nginx.*\|.*python.*\|.*postmaster.*\|.*mongod\|.*init//') - if [ "$rv" != "Unbound" ] && [ "$rv" != "" ]; then - used+=("$rv") + local PKG_VER="" + + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + + + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} +system_install_runtime_packages() +{ + + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + + iproute2 + gnupg + apt-transport-https + apache2-utils + ca-certificates + + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" fi - done - # If any used ports were found, display helpful message and exit - if [ ${#used[@]} -gt 0 ]; then - printf "\nNot all required TCP ports are available. ST2 and related services will fail to start.\n\n" - echo "The following ports are in use by the specified pid/process and need to be stopped:" - for port_pid_process in "${used[@]}" + service_config_path="" + for cfgdir in "/etc/sysconfig" "/etc/default" do - echo " $port_pid_process" + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi done - echo "" - exit 1 - fi - # CHECK 2: Ensure there is enough space at /var/lib/mongodb - VAR_SPACE=`df -Pk /var/lib | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{print $4}'` - if [ ${VAR_SPACE} -lt 358400 ]; then - echo "" - echo "MongoDB requires at least 350MB free in /var/lib/mongodb" - echo "There is not enough space for MongoDB. It will fail to start." - echo "Please, add some space to /var or clean it up." - exit 1 - fi + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done } -generate_random_passwords() { - # Generate random password used for MongoDB user authentication - ST2_MONGODB_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') - # Generate random password used for RabbitMQ user authentication - ST2_RABBITMQ_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') +system_port_status() +{ + # + # + # + + # + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" } -configure_st2_user () { - # Create an SSH system user (default `stanley` user may be already created) - if (! id stanley 2>/dev/null); then - sudo useradd stanley - fi +system_check_resources() +{ + + PORT_TEST=$( + cat <> ${SYSTEM_HOME}/.ssh/authorized_keys" - fi + Password credentials have been saved to /root/st2_credentials. + Please store them in a secure location and delete the file. - sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys - sudo chmod 0700 ${SYSTEM_HOME}/.ssh - sudo chown -R stanley:stanley ${SYSTEM_HOME} - # Enable passwordless sudo - local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" - if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" - fi +EOF +} - sudo chmod 0440 /etc/sudoers.d/st2 +write_passwords() +{ + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} - # Disable requiretty for all users - sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo } -configure_st2_cli_config() { - # Configure CLI config (write credentials for the root user and user which ran the script) - ROOT_USER="root" - CURRENT_USER=$(whoami) +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} - ROOT_HOME=$(eval echo ~${ROOT_USER}) - : "${HOME:=$(eval echo ~${CURRENT_USER})}" - ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" - ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + echo "${OS_VERSION_CODENAME}" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + local PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" + curl -sSL -k -o "${PACKAGE_FILENAME}" "${PACKAGE_URL}" + sudo dpkg --install --force-depends "${PACKAGE_FILENAME}" + sudo apt install --yes --fix-broken + rm "${PACKAGE_FILENAME}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}=${VERSION}" +} - CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" - CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" - if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then - sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} - fi +st2_install_dev_build() +{ + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" - sudo sh -c "cat < ${ROOT_USER_CLI_CONFIG_PATH} + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi + + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" +} + +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="/etc/st2/st2.conf" + + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" + + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" + + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} + + +st2_configure_authentication() { + local ST2_CFGFILE="/etc/st2/st2.conf" + + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done +} + + +st2_configure_user() +{ + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi + + SYSTEM_HOME=$(echo ~stanley) + + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi + + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi + + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi + + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} + + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + + sudo chmod 0440 /etc/sudoers.d/st2 + + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers +} + + +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) + + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" + + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} [credentials] username = ${USERNAME} password = ${PASSWORD} -EOT" +EOF" - # Write config for root user - if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then - return - fi + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return + fi - # Write config for current user (in case current user != root) - if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then - sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} - fi + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + fi - sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} [credentials] username = ${USERNAME} password = ${PASSWORD} -EOT" +EOF" - # Fix the permissions - sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} } -generate_symmetric_crypto_key_for_datastore() { - DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" - DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" - - sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" - # If the file ${DATASTORE_ENCRYPTION_KEY_PATH} exists and is not empty, then do not generate - # a new key. st2-generate-symmetric-crypto-key fails if the key file already exists. - if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then - sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} - fi + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - # Make sure only st2 user can read the file - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEY_PATH} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEY_PATH} + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} + fi - # set path to the key file in the config - sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} - # NOTE: We need to restart all the affected services so they pick the key and load it in memory - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2sensorcontainer - sudo st2ctl restart-component st2workflowengine - sudo st2ctl restart-component st2actionrunner + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done } -verify_st2() { - st2 --version - st2 -h - - st2 auth $USERNAME -p $PASSWORD - # A shortcut to authenticate and export the token - export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) +st2_verification() +{ + echo.info "Check version" + st2 --version - # List the actions from a 'core' pack - st2 action list --pack=core + echo.info "Check help" + st2 -h - # Run a local shell command - st2 run core.local -- date -R + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) - # See the execution results - st2 execution list + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core - # Fire a remote comand via SSH (Requires passwordless SSH) - st2 run core.remote hosts='127.0.0.1' -- uname -a + echo.info "Check local shell command" + st2 run core.local -- date -R - # Install a pack - st2 pack install st2 -} + echo.info "Check execution list" + st2 execution list + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a -ok_message() { - echo "" - echo "" - echo "███████╗████████╗██████╗ ██████╗ ██╗ ██╗"; - echo "██╔════╝╚══██╔══╝╚════██╗ ██╔═══██╗██║ ██╔╝"; - echo "███████╗ ██║ █████╔╝ ██║ ██║█████╔╝ "; - echo "╚════██║ ██║ ██╔═══╝ ██║ ██║██╔═██╗ "; - echo "███████║ ██║ ███████╗ ╚██████╔╝██║ ██╗"; - echo "╚══════╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝"; - echo "" - echo " st2 is installed and ready to use." - echo "" - echo "Head to https://YOUR_HOST_IP/ to access the WebUI" - echo "" - echo "Don't forget to dive into our documentation! Here are some resources" - echo "for you:" - echo "" - echo "* Documentation - https://docs.stackstorm.com" - echo "* Pack Exchange - https://exchange.stackstorm.org/" - echo "" - echo "Thanks for installing StackStorm! Come visit us in our Slack Channel" - echo "and tell us how it's going. We'd love to hear from you!" - echo "http://stackstorm.com/community-signup" + echo.info "Check pack installation" + st2 pack install st2 } - -fail() { - echo "############### ERROR ###############" - echo "# Failed on $STEP #" - echo "#####################################" - exit 2 +nodejs_configure_repository() +{ + curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - } +st2chatops_install() +{ + nodejs_configure_repository + pkg_install nodejs -install_net_tools() { - # Install netstat - sudo apt install -y net-tools + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} } -install_st2_dependencies() { - # Silence debconf prompt, raised during some dep installations. This will be passed to sudo via 'env_keep'. - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update - - sudo apt-get install -y curl +st2chatops_configure() +{ + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} - # Various other dependencies needed by st2 and installer script - sudo apt-get install -y crudini +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "nginx" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" } -install_rabbitmq() { - # Install pre-req - sudo apt-get install -y gnupg apt-transport-https +st2web_install() +{ + nginx_configure_repo + pkg_meta_update - # Team RabbitMQ's main signing key - curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null - # CloudSmith Erlang Repository - curl -1sLf "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/gpg.E495BB49CC4BBE5B.key" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.dl.rabbitmq.erlang.gpg > /dev/null - # CloudSmith RabbitMQ repository - curl -1sLf "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/gpg.9F4587F226208342.key" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.dl.rabbitmq.gpg > /dev/null + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} - # Add apt repositories maintained by Team RabbitMQ - sudo tee /etc/apt/sources.list.d/rabbitmq.list <> /etc/rabbitmq/rabbitmq-env.conf' - sudo systemctl restart rabbitmq-server +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/apt/${OS_ID}" \ + "${OS_VERSION_CODENAME}/mongodb-org/7.0" \ + "multiverse" \ + "mongodb-org-7.0-key" \ + "https://www.mongodb.org/static/pgp/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongodb" + local DB_PATH="/var/lib/mongodb" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" +} + +mongodb_write_configuration() +{ + local MONGODB_USER="$1" + local DB_PATH="$2" + local LOG_PATH="$3" + local CFGFILE="/etc/mongod.conf" + + TMP=$(cat <${CFGFILE}" } -install_mongodb() { - # Add key and repo for the MongoDB 4.4 (Ubuntu Focal) - MONGO_VERSION="4.4" +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} - wget -qO - https://www.mongodb.org/static/pgp/server-${MONGO_VERSION}.asc | sudo apt-key add - - echo "deb http://repo.mongodb.org/apt/ubuntu ${SUBTYPE}/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list +mongodb_install() +{ + local MONGODB_PKG=mongodb-org - # Install MongoDB - sudo apt-get update - sudo apt-get install -y mongodb-org + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration - # Enable and restart - sudo systemctl enable mongod - sudo systemctl start mongod + sudo systemctl enable mongod + sudo systemctl start mongod - # Wait for service to come up before attempt to create user - sleep 10 + sleep 10 - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <> /etc/mongod.conf' - - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod - + sudo sed -ri 's/^ authorization: disabled$/ authorization: enabled/g' /etc/mongod.conf + sudo systemctl restart mongod } +rabbitmq_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + pkg_install policycoreutils-python-utils -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo apt-get install -y redis-server -} + sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 -get_full_pkg_versions() { - if [[ "$VERSION" != '' ]]; - then - local ST2_VER=$(apt-cache show st2 | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2_VER" ]]; then - echo "Could not find requested version of StackStorm!!!" - sudo apt-cache policy st2 - exit 3 + sudo setsebool -P httpd_can_network_connect 1 fi +} - local ST2WEB_VER=$(apt-cache show st2web | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2WEB_VER" ]]; then - echo "Could not find requested version of st2web." - sudo apt-cache policy st2web - exit 3 - fi +rabbitmq_install() +{ + local RABBITMQ_PKG=rabbitmq-server - local ST2CHATOPS_VER=$(apt-cache show st2chatops | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2CHATOPS_VER" ]]; then - echo "Could not find requested version of st2chatops." - sudo apt-cache policy st2chatops - exit 3 + if [[ $INSTALL_RABBITMQ -eq 0 ]]; then + echo.info "Skip RabbitMQ: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$RABBITMQ_PKG"; then + echo.info "Skip RabbitMQ: Package is already present on the system." + return fi - ST2_PKG_VERSION="=${ST2_VER}" - ST2WEB_PKG_VERSION="=${ST2WEB_VER}" - ST2CHATOPS_PKG_VERSION="=${ST2CHATOPS_VER}" - echo "##########################################################" - echo "#### Following versions of packages will be installed ####" - echo "st2${ST2_PKG_VERSION}" - echo "st2web${ST2WEB_PKG_VERSION}" - echo "st2chatops${ST2CHATOPS_PKG_VERSION}" - echo "##########################################################" - fi -} + repo_add_gpg_key "com.rabbitmq.team.gpg" "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" + + repo_definition "erlang" \ + "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-erlang/deb/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "erlang-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key" + repo_definition "rabbitmq-server" \ + "https://ppa1.rabbitmq.com/rabbitmq/rabbitmq-server/deb/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "rabbitmq-server-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key" + + local PKGS=( + erlang-base + erlang-asn1 + erlang-crypto + erlang-eldap + erlang-ftp + erlang-inets + erlang-mnesia + erlang-os-mon + erlang-parsetools + erlang-public-key + erlang-runtime-tools + erlang-snmp + erlang-ssl + erlang-syntax-tools + erlang-tftp + erlang-tools + erlang-xmerl + "$RABBITMQ_PKG" + ) + cat </etc/apt/preferences.d/erlang.pref +Package: erlang* +Pin: origin ppa1.rabbitmq.com +Pin-Priority: 1001 +EOF -install_st2() { - # Following script adds a repo file, registers gpg key and runs apt-get update - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.deb.sh | sudo bash - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo apt-get install -y st2${ST2_PKG_VERSION} - else - sudo apt-get install -y jq + pkg_meta_update + pkg_install ${PKGS[@]} - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "${SUBTYPE}" "st2_.*.deb") - PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" - curl -sSL -k -o ${PACKAGE_FILENAME} ${PACKAGE_URL} - sudo dpkg -i --force-depends ${PACKAGE_FILENAME} - sudo apt-get install -yf - rm ${PACKAGE_FILENAME} - fi + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" + repo_definition "redis" \ + "https://packages.redis.io/deb" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "redis-key" \ + "https://packages.redis.io/gpg" + local REDIS_SERVICE=redis-server + + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi - sudo st2ctl start - sudo st2ctl reload --register-all + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" } +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done -configure_st2_authentication() { - # Install htpasswd tool for editing ini files - sudo apt-get install -y apache2-utils +trap 'fail' EXIT - # Create a user record in a password file. - sudo echo "${PASSWORD}" | sudo htpasswd -i /etc/st2/htpasswd $USERNAME +step "Setup runtime arguments" +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' +step "Install required runtime packages" +system_install_runtime_packages - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} +step "Check storage capacity and network ports" +system_check_resources +step "Configure HTTP Proxy" +system_configure_proxy -install_st2web() { - # Add key and repo for the latest stable nginx - sudo apt-key adv --fetch-keys http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/apt/sources.list.d/nginx.list -deb http://nginx.org/packages/ubuntu/ ${SUBTYPE} nginx -deb-src http://nginx.org/packages/ubuntu/ ${SUBTYPE} nginx -EOT" - sudo apt-get update +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords - # Install st2web and nginx - sudo apt-get install -y st2web${ST2WEB_PKG_VERSION} nginx +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ - Technology/CN=$(hostname)" +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ +step "Install key/value store (Redis)" +redis_install - sudo service nginx restart -} +step "Install st2 (StackStorm)" +st2_install -install_st2chatops() { - # Update certificates - sudo apt-get update - sudo apt-get install -y ca-certificates +step "Configure st2 system user account" +st2_configure_user - # Add NodeJS 20 repo - curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - +step "Configure st2 authentication" +st2_configure_authentication - # Install st2chatops - sudo apt-get install -y st2chatops${ST2CHATOPS_PKG_VERSION} -} +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env +step "Verify StackStorm installation" +st2_verification - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env +step "Install Web Interface (st2web)" +st2web_install - sudo service st2chatops restart - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/deb.html#setup-chatops" - echo "########################################################" - fi -} +step "Install ChatOps bot (st2chatops)" +st2chatops_install +step "Configure st2chatops" +st2chatops_configure -## Let's do this! - -trap 'fail' EXIT -STEP="Setup args" && setup_args $@ -STEP="Install net-tools" && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP="Generate random password" && generate_random_passwords -STEP="Configure Proxy" && configure_proxy -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web - -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops trap - EXIT ok_message diff --git a/scripts/st2bootstrap-deb.template.sh b/scripts/st2bootstrap-deb.template.sh deleted file mode 100644 index fd801009..00000000 --- a/scripts/st2bootstrap-deb.template.sh +++ /dev/null @@ -1,433 +0,0 @@ -set -eu - -HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} -VERSION='' -RELEASE='stable' -REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' -ST2WEB_PKG_VERSION='' -ST2CHATOPS_PKG_VERSION='' -DEV_BUILD='' -USERNAME='' -PASSWORD='' -U16_ADD_INSECURE_PY3_PPA=0 -SUBTYPE=`lsb_release -cs` - -if [[ "$SUBTYPE" != 'focal' ]]; then - echo "Unsupported ubuntu codename ${SUBTYPE}. Please use Ubuntu 20.04 (focal) as base system!" - exit 2 -fi - -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - # Provide flag to enable installing Python3 from 3rd party insecure PPA for Ubuntu Xenial - # TODO: Remove once Ubuntu Xenial is dropped - --u16-add-insecure-py3-ppa) - U16_ADD_INSECURE_PY3_PPA=1 - shift - ;; - *) - # unknown option - ;; - esac - done - - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - - if [[ "$VERSION" != '' ]]; then - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 - fi - - if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing StackStorm $RELEASE $VERSION " - echo "########################################################" - - if [ "$REPO_TYPE" == "staging" ]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [ "$DEV_BUILD" != '' ]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [ "${PASSWORD}" = '' ]; then - echo "Password cannot be empty." - exit 1 - fi - fi - -} - - -# include:includes/common.sh - -install_net_tools() { - # Install netstat - sudo apt install -y net-tools -} - -install_st2_dependencies() { - # Silence debconf prompt, raised during some dep installations. This will be passed to sudo via 'env_keep'. - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update - - sudo apt-get install -y curl - - # Various other dependencies needed by st2 and installer script - sudo apt-get install -y crudini -} - -install_rabbitmq() { - # Install pre-req - sudo apt-get install -y gnupg apt-transport-https - - # Team RabbitMQ's main signing key - curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null - # CloudSmith Erlang Repository - curl -1sLf "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/gpg.E495BB49CC4BBE5B.key" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.dl.rabbitmq.erlang.gpg > /dev/null - # CloudSmith RabbitMQ repository - curl -1sLf "https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/gpg.9F4587F226208342.key" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.dl.rabbitmq.gpg > /dev/null - - # Add apt repositories maintained by Team RabbitMQ - sudo tee /etc/apt/sources.list.d/rabbitmq.list <> /etc/rabbitmq/rabbitmq-env.conf' - - sudo systemctl restart rabbitmq-server - -} - -install_mongodb() { - # Add key and repo for the MongoDB 4.4 (Ubuntu Focal) - MONGO_VERSION="4.4" - - wget -qO - https://www.mongodb.org/static/pgp/server-${MONGO_VERSION}.asc | sudo apt-key add - - echo "deb http://repo.mongodb.org/apt/ubuntu ${SUBTYPE}/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list - - # Install MongoDB - sudo apt-get update - sudo apt-get install -y mongodb-org - - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf - - # Enable and restart - sudo systemctl enable mongod - sudo systemctl start mongod - - # Wait for service to come up before attempt to create user - sleep 10 - - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <> /etc/mongod.conf' - - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod - - -} - -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo apt-get install -y redis-server -} - -get_full_pkg_versions() { - if [[ "$VERSION" != '' ]]; - then - local ST2_VER=$(apt-cache show st2 | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2_VER" ]]; then - echo "Could not find requested version of StackStorm!!!" - sudo apt-cache policy st2 - exit 3 - fi - - local ST2WEB_VER=$(apt-cache show st2web | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2WEB_VER" ]]; then - echo "Could not find requested version of st2web." - sudo apt-cache policy st2web - exit 3 - fi - - local ST2CHATOPS_VER=$(apt-cache show st2chatops | grep Version | awk '{print $2}' | grep ^${VERSION//./\\.} | sort --version-sort | tail -n 1) - if [[ -z "$ST2CHATOPS_VER" ]]; then - echo "Could not find requested version of st2chatops." - sudo apt-cache policy st2chatops - exit 3 - fi - - ST2_PKG_VERSION="=${ST2_VER}" - ST2WEB_PKG_VERSION="=${ST2WEB_VER}" - ST2CHATOPS_PKG_VERSION="=${ST2CHATOPS_VER}" - echo "##########################################################" - echo "#### Following versions of packages will be installed ####" - echo "st2${ST2_PKG_VERSION}" - echo "st2web${ST2WEB_PKG_VERSION}" - echo "st2chatops${ST2CHATOPS_PKG_VERSION}" - echo "##########################################################" - fi -} - -install_st2() { - # Following script adds a repo file, registers gpg key and runs apt-get update - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.deb.sh | sudo bash - - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo apt-get install -y st2${ST2_PKG_VERSION} - else - sudo apt-get install -y jq - - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "${SUBTYPE}" "st2_.*.deb") - PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" - curl -sSL -k -o ${PACKAGE_FILENAME} ${PACKAGE_URL} - sudo dpkg -i --force-depends ${PACKAGE_FILENAME} - sudo apt-get install -yf - rm ${PACKAGE_FILENAME} - fi - - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" - - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" - - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" - - sudo st2ctl start - sudo st2ctl reload --register-all -} - - -configure_st2_authentication() { - # Install htpasswd tool for editing ini files - sudo apt-get install -y apache2-utils - - # Create a user record in a password file. - sudo echo "${PASSWORD}" | sudo htpasswd -i /etc/st2/htpasswd $USERNAME - - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' - - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} - - -install_st2web() { - # Add key and repo for the latest stable nginx - sudo apt-key adv --fetch-keys http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/apt/sources.list.d/nginx.list -deb http://nginx.org/packages/ubuntu/ ${SUBTYPE} nginx -deb-src http://nginx.org/packages/ubuntu/ ${SUBTYPE} nginx -EOT" - sudo apt-get update - - # Install st2web and nginx - sudo apt-get install -y st2web${ST2WEB_PKG_VERSION} nginx - - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ - Technology/CN=$(hostname)" - - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - - sudo service nginx restart -} - -install_st2chatops() { - # Update certificates - sudo apt-get update - sudo apt-get install -y ca-certificates - - # Add NodeJS 20 repo - curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - - - # Install st2chatops - sudo apt-get install -y st2chatops${ST2CHATOPS_PKG_VERSION} -} - -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env - - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env - - sudo service st2chatops restart - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/deb.html#setup-chatops" - echo "########################################################" - fi -} - - -## Let's do this! - -trap 'fail' EXIT -STEP="Setup args" && setup_args $@ -STEP="Install net-tools" && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP="Generate random password" && generate_random_passwords -STEP="Configure Proxy" && configure_proxy -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web - -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops -trap - EXIT - -ok_message diff --git a/scripts/st2bootstrap-el8.sh b/scripts/st2bootstrap-el8.sh index 87303c26..97ea8052 100644 --- a/scripts/st2bootstrap-el8.sh +++ b/scripts/st2bootstrap-el8.sh @@ -1,567 +1,904 @@ #!/usr/bin/env bash -# NOTE: This file is automatically generated by the tools/generate_final_installer_scripts.py -# script using the template file and common include files in scripts/includes/*.sh. # -# DO NOT EDIT MANUALLY. # -# Please edit corresponding template file and include files. -set -eu +set -e -u +x HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} VERSION='' RELEASE='stable' REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' DEV_BUILD='' USERNAME='' PASSWORD='' ST2_PKG='st2' ST2WEB_PKG='st2web' ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 -is_rhel() { - return $(cat /etc/os-release | grep 'ID="rhel"') -} +declare -A INSTALL_TYPE=() -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - *) - # unknown option - ;; - esac - done +source <(sed 's/^/OS_/g' /etc/os-release) + +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + + echo "${OUTPUT}" +} +function cecho() +{ + if [[ "$1" == "-n" ]]; then + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} +setup_install_parameters() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + if [[ -n "$DEV_BUILD" ]]; then + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - if [[ "$VERSION" != '' ]]; then +setup_check_version() +{ + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing st2 $RELEASE $VERSION " - echo "########################################################" - - if [[ "$REPO_TYPE" == "staging" ]]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [[ "$DEV_BUILD" != '' ]]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [[ "${PASSWORD}" = '' ]]; then - echo "Password cannot be empty." - exit 1 + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi fi - fi -} - - -function configure_proxy() { - # Allow bypassing 'proxy' env vars via sudo - local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' - if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" - fi - - # Configure proxy env vars for 'st2api', 'st2actionrunner' and 'st2chatops' system configs - # See: https://docs.stackstorm.com/packs.html#installing-packs-from-behind-a-proxy - local service_config_path=$(hash apt-get >/dev/null 2>&1 && echo '/etc/default' || echo '/etc/sysconfig') - for service in st2api st2actionrunner st2chatops; do - service_config="${service_config_path}/${service}" - # create file if doesn't exist yet - sudo test -e ${service_config} || sudo touch ${service_config} - for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do - # delete line from file if specific proxy env var is unset - if sudo test -z "${!env_var:-}"; then - sudo sed -i "/^${env_var}=/d" ${service_config} - # add proxy env var if it doesn't exist yet - elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then - sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" - # modify existing proxy env var value - elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then - sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} - fi - done - done + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" } -function get_package_url() { - # Retrieve direct package URL for the provided dev build, subtype and package name regex. - DEV_BUILD=$1 # Repo name and build number - / (e.g. st2/5646) - DISTRO=$2 # Distro name (e.g. focal,jammy,el8,el9) - PACKAGE_NAME_REGEX=$3 +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} +pkg_install() +{ + + sudo dnf -y install $@ +} - PACKAGES_METADATA=$(curl -sSL -q https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts) +pkg_meta_update() +{ + + sudo dnf -y check-update + +} + +pkg_is_installed() +{ + PKG="$1" + + sudo rpm -q "$PKG" | grep -qE "^${PKG}" + +} + + +pkg_get_latest_version() +{ + local PKG="$1" # st2 + local VERSION="$2" # 3.9dev + LATEST=$(repoquery -y --nvr --show-duplicates "$PKG" | grep -F "${PKG}-${VERSION}" | sort --version-sort | tail -n 1) + echo "${LATEST#*-}" +} - if [ -z "${PACKAGES_METADATA}" ]; then - echo "Failed to retrieve packages metadata from https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" 1>&2 - return 2 - fi - PACKAGES_URLS="$(echo ${PACKAGES_METADATA} | jq -r '.[].url')" - PACKAGE_URL=$(echo "${PACKAGES_URLS}" | egrep "${DISTRO}/${PACKAGE_NAME_REGEX}") +repo_add_gpg_key() +{ + KEY_NAME="$1" + KEY_URL="$2" + rpm --import "${KEY_URL}" +} + - if [ -z "${PACKAGE_URL}" ]; then - echo "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" 1>&2 - echo "Circle CI response: ${PACKAGES_METADATA}" 1>&2 - return 2 - fi +repo_definition() +{ + REPO_NAME="$1" + REPO_URL="$2" + KEY_NAME="$3" + KEY_URL="$4" + REPO_PATH="/etc/yum.repos.d/" - echo ${PACKAGE_URL} + cat <"${REPO_PATH}/${REPO_NAME}.repo" +[${REPO_NAME}] +name=${REPO_NAME} +baseurl=${REPO_URL} +repo_gpgcheck=1 +enabled=1 +gpgkey=${KEY_URL} +gpgcheck=0 +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +pkg_gpgcheck=1 +autorefresh=1 +type=rpm-md +EOF } -function port_status() { - # If the specified tcp4 port is bound, then return the "port pid/procname", - # else if a pipe command fails, return "Unbound", - # else return "". - # - # Please note that all return values end with a newline. - # - # Use netstat and awk to get a list of all the tcp4 sockets that are in the LISTEN state, - # matching the specified port. - # - # The `netstat -tunlp --inet` command is assumed to output data in the following format: - # Active Internet connections (only servers) - # Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name - # tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7506/httpd - # - # The awk command prints the 4th and 7th columns of any line matching both the following criteria: - # 1) The 4th column contains the port passed to port_status() (i.e., $1) - # 2) The 6th column contains "LISTEN" - # - # Sample output: - # 0.0.0.0:25000 7506/sshd - ret=$(sudo netstat -tunlp --inet | awk -v port=":$1$" '$4 ~ port && $6 ~ /LISTEN/ { print $4 " " $7 }' || echo 'Unbound'); - echo "$ret"; +pkg_get_versions() +{ + dnf -y info --showduplicates "$1" } -check_st2_host_dependencies() { - # CHECK 1: Determine which, if any, of the required ports are used by an existing process. +repo_clean_meta() +{ + dnf -y clean metadata + dnf -y clean dbcache + dnf -y clean all +} + - # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) - # and st2 (9100-9102). +repo_pkg_availability() { + local PKG="$1" + local VERSION="$2" - declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") - declare -a used=() + local PKG_VER="" + + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + - for i in "${ports[@]}" - do - rv=$(port_status $i | sed 's/.*-$\|.*systemd\|.*beam.smp.*\|.*epmd\|.*st2.*\|.*nginx.*\|.*python.*\|.*postmaster.*\|.*mongod\|.*init//') - if [ "$rv" != "Unbound" ] && [ "$rv" != "" ]; then - used+=("$rv") + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} +system_install_runtime_packages() +{ + + if ! pkg_is_installed epel-release; then + pkg_install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + fi + + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + yum-utils + iproute + gnupg2 + httpd-tools + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" fi - done - # If any used ports were found, display helpful message and exit - if [ ${#used[@]} -gt 0 ]; then - printf "\nNot all required TCP ports are available. ST2 and related services will fail to start.\n\n" - echo "The following ports are in use by the specified pid/process and need to be stopped:" - for port_pid_process in "${used[@]}" + service_config_path="" + for cfgdir in "/etc/sysconfig" "/etc/default" do - echo " $port_pid_process" + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi done - echo "" - exit 1 - fi - # CHECK 2: Ensure there is enough space at /var/lib/mongodb - VAR_SPACE=`df -Pk /var/lib | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{print $4}'` - if [ ${VAR_SPACE} -lt 358400 ]; then - echo "" - echo "MongoDB requires at least 350MB free in /var/lib/mongodb" - echo "There is not enough space for MongoDB. It will fail to start." - echo "Please, add some space to /var or clean it up." - exit 1 - fi + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done } -generate_random_passwords() { - # Generate random password used for MongoDB user authentication - ST2_MONGODB_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') - # Generate random password used for RabbitMQ user authentication - ST2_RABBITMQ_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') +system_port_status() +{ + # + # + # + + # + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" } -configure_st2_user () { - # Create an SSH system user (default `stanley` user may be already created) - if (! id stanley 2>/dev/null); then - sudo useradd stanley - fi +system_check_resources() +{ + + PORT_TEST=$( + cat <> ${SYSTEM_HOME}/.ssh/authorized_keys" - fi +system_generate_password() +{ + local LEN="$1" + head /dev/urandom | tr -dc A-Za-z0-9 | head -c $LEN; echo '' +} - sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys - sudo chmod 0700 ${SYSTEM_HOME}/.ssh - sudo chown -R stanley:stanley ${SYSTEM_HOME} - # Enable passwordless sudo - local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" - if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" - fi +ok_message() +{ + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} - ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" - ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo +} - CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" - CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" - if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then - sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} - fi +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} - sudo sh -c "cat < ${ROOT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/el/8/\$basearch/" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + echo "el8" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + pkg_install "${PACKAGE_URL}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}-${VERSION}" +} - # Write config for root user - if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then - return - fi - # Write config for current user (in case current user != root) - if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then - sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} - fi +st2_install_dev_build() +{ + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" - sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi - # Fix the permissions - sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" } +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="/etc/st2/st2.conf" -generate_symmetric_crypto_key_for_datastore() { - DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" - DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" - sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" - # If the file ${DATASTORE_ENCRYPTION_KEY_PATH} exists and is not empty, then do not generate - # a new key. st2-generate-symmetric-crypto-key fails if the key file already exists. - if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then - sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} - fi + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} - # Make sure only st2 user can read the file - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEY_PATH} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEY_PATH} - # set path to the key file in the config - sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} +st2_configure_authentication() { + local ST2_CFGFILE="/etc/st2/st2.conf" - # NOTE: We need to restart all the affected services so they pick the key and load it in memory - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2sensorcontainer - sudo st2ctl restart-component st2workflowengine - sudo st2ctl restart-component st2actionrunner + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done } -verify_st2() { - st2 --version - st2 -h +st2_configure_user() +{ + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi - st2 auth $USERNAME -p $PASSWORD - # A shortcut to authenticate and export the token - export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) + SYSTEM_HOME=$(echo ~stanley) - # List the actions from a 'core' pack - st2 action list --pack=core + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi - # Run a local shell command - st2 run core.local -- date -R + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi - # See the execution results - st2 execution list + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi - # Fire a remote comand via SSH (Requires passwordless SSH) - st2 run core.remote hosts='127.0.0.1' -- uname -a + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} - # Install a pack - st2 pack install st2 -} + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + sudo chmod 0440 /etc/sudoers.d/st2 -ok_message() { - echo "" - echo "" - echo "███████╗████████╗██████╗ ██████╗ ██╗ ██╗"; - echo "██╔════╝╚══██╔══╝╚════██╗ ██╔═══██╗██║ ██╔╝"; - echo "███████╗ ██║ █████╔╝ ██║ ██║█████╔╝ "; - echo "╚════██║ ██║ ██╔═══╝ ██║ ██║██╔═██╗ "; - echo "███████║ ██║ ███████╗ ╚██████╔╝██║ ██╗"; - echo "╚══════╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝"; - echo "" - echo " st2 is installed and ready to use." - echo "" - echo "Head to https://YOUR_HOST_IP/ to access the WebUI" - echo "" - echo "Don't forget to dive into our documentation! Here are some resources" - echo "for you:" - echo "" - echo "* Documentation - https://docs.stackstorm.com" - echo "* Pack Exchange - https://exchange.stackstorm.org/" - echo "" - echo "Thanks for installing StackStorm! Come visit us in our Slack Channel" - echo "and tell us how it's going. We'd love to hear from you!" - echo "http://stackstorm.com/community-signup" + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers } -fail() { - echo "############### ERROR ###############" - echo "# Failed on $STEP #" - echo "#####################################" - exit 2 -} +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) -install_yum_utils() { - # We need repoquery tool to get package_name-package_ver-package_rev in RPM based distros - # if we don't want to construct this string manually using yum info --show-duplicates and - # doing a bunch of sed awk magic. Problem is this is not installed by default on all images. - sudo yum install -y yum-utils -} + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" -get_full_pkg_versions() { - if [ "$VERSION" != '' ]; - then - local RHMAJVER=`cat /etc/redhat-release | sed 's/[^0-9.]*\([0-9.]\).*/\1/'` - local YES_FLAG="" - if [ "$RHMAJVER" -ge "8" ]; then - # RHEL 8 and newer, you need "-y" flag to avoid being prompted to confirm "yes" - local YES_FLAG="-y" + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} fi - local ST2_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2 | grep -F st2-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2_VER" ]; then - echo "Could not find requested version of st2!!!" - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2 - exit 3 + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return fi - ST2_PKG=${ST2_VER} - local ST2WEB_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2web | grep -F st2web-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2WEB_VER" ]; then - echo "Could not find requested version of st2web." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2web - exit 3 + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} fi - ST2WEB_PKG=${ST2WEB_VER} - local ST2CHATOPS_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops | grep -F st2chatops-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2CHATOPS_VER" ]; then - echo "Could not find requested version of st2chatops." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops - exit 3 + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} +} + + +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} fi - ST2CHATOPS_PKG=${ST2CHATOPS_VER} - echo "##########################################################" - echo "#### Following versions of packages will be installed ####" - echo "${ST2_PKG}" - echo "${ST2WEB_PKG}" - echo "${ST2CHATOPS_PKG}" - echo "##########################################################" - fi + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done } +st2_verification() +{ + echo.info "Check version" + st2 --version + + echo.info "Check help" + st2 -h + + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) -# Note that default SELINUX policies for RHEL8 differ with Rocky8. Rocky8 is more permissive by default -# Note that depending on distro assembly/settings you may need more rules to change -# Apply these changes OR disable selinux in /etc/selinux/config (manually) -adjust_selinux_policies() { - if getenforce | grep -q 'Enforcing'; then - # SELINUX management tools, not available for some minimal installations - sudo yum install -y policycoreutils-python-utils + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core - # Allow rabbitmq to use '25672' port, otherwise it will fail to start - sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 + echo.info "Check local shell command" + st2 run core.local -- date -R - # Allow network access for nginx - sudo setsebool -P httpd_can_network_connect 1 - fi -} - -install_net_tools() { - # Install netstat - sudo yum install -y net-tools -} - -install_st2_dependencies() { - # RabbitMQ on RHEL8 requires module(perl:5.26 - if is_rhel; then - sudo yum -y module enable perl:5.26 - fi + echo.info "Check execution list" + st2 execution list - is_epel_installed=$(rpm -qa | grep epel-release || true) - if [[ -z "$is_epel_installed" ]]; then - sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - fi + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a - # Various other dependencies needed by st2 and installer script - sudo yum -y install crudini + echo.info "Check pack installation" + st2 pack install st2 +} +nodejs_configure_repository() +{ + curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo -E bash - } -install_rabbitmq() { - # Install erlang from rabbitmq/erlang as need newer version - # than available in epel. - curl -sL https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash - sudo yum -y install erlang-25* - # Install rabbit from packagecloud - curl -sL https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash - sudo yum makecache -y --disablerepo='*' --enablerepo='rabbitmq_rabbitmq-server' - - sudo yum -y install curl rabbitmq-server +st2chatops_install() +{ + nodejs_configure_repository + pkg_install nodejs - # Configure RabbitMQ to listen on localhost only - sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} +} - sudo systemctl start rabbitmq-server - sudo systemctl enable rabbitmq-server +st2chatops_configure() +{ + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/rhel/8/x86_64/" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" - sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" - sudo rabbitmqctl delete_user guest - sudo rabbitmqctl set_user_tags stackstorm administrator - sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" } -install_mongodb() { +st2web_install() +{ + nginx_configure_repo + pkg_meta_update - # Add key and repo for the latest stable MongoDB (4.0) - sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo -[mongodb-org-4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/4.0/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc -EOT" + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} + + sudo mkdir -p /etc/ssl/st2 + sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ + -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ + Technology/CN=$(hostname)" - sudo yum -y install mongodb-org + sudo rm -f /etc/nginx/conf.d/default.conf - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf + sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak + sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak >/etc/nginx/nginx.conf + sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf + sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf + - sudo systemctl start mongod - sudo systemctl enable mongod + sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - sleep 5 + sudo systemctl enable nginx + sudo systemctl restart nginx +} +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/yum/redhat/8/mongodb-org/7.0/x86_64/" \ + "mongodb-org-7.0-key" \ + "https://pgp.mongodb.com/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongod" + local DB_PATH="/var/lib/mongo" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" + mongodb_adjust_selinux_policies +} - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <${CFGFILE}" +} + +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} + +mongodb_install() +{ + local MONGODB_PKG=mongodb-org + + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi + + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration + + sudo systemctl enable mongod + sudo systemctl start mongod + + sleep 10 + + mongosh <> /etc/mongod.conf' + sudo sed -ri 's/^ authorization: disabled$/ authorization: enabled/g' /etc/mongod.conf - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod + sudo systemctl restart mongod } +rabbitmq_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + pkg_install policycoreutils-python-utils + + sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo yum install -y redis - sudo systemctl start redis - sudo systemctl enable redis + sudo setsebool -P httpd_can_network_connect 1 + fi } -install_st2() { - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash +rabbitmq_install() +{ + local RABBITMQ_PKG=rabbitmq-server - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo yum -y install ${ST2_PKG} - else - sudo yum -y install jq + if [[ $INSTALL_RABBITMQ -eq 0 ]]; then + echo.info "Skip RabbitMQ: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$RABBITMQ_PKG"; then + echo.info "Skip RabbitMQ: Package is already present on the system." + return + fi + repo_definition "erlang" \ + "https://yum1.rabbitmq.com/erlang/el/8/\$basearch" \ + "erlang-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key" + repo_definition "rabbitmq-server" \ + "https://yum2.rabbitmq.com/rabbitmq/el/8/\$basearch" \ + "rabbitmq-server-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc" + repo_definition "rabbitmq-server-noarch" \ + "https://yum2.rabbitmq.com/rabbitmq/el/8/noarch" \ + "rabbitmq-server-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc" + + rabbitmq_adjust_selinux_policies + + local PKGS=( + erlang + "$RABBITMQ_PKG" + ) + + + pkg_meta_update + pkg_install ${PKGS[@]} + + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server + + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi + local REDIS_SERVICE=redis + + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "el8" "st2-.*.rpm") - sudo yum -y install ${PACKAGE_URL} - fi + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" +} - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" +trap 'fail' EXIT - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" +step "Setup runtime arguments" +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password - sudo st2ctl start - sudo st2ctl reload --register-all -} +step "Install required runtime packages" +system_install_runtime_packages +step "Check storage capacity and network ports" +system_check_resources -configure_st2_authentication() { - # Install htpasswd tool - sudo yum -y install httpd-tools +step "Configure HTTP Proxy" +system_configure_proxy - # Create a user record in a password file. - echo $PASSWORD | sudo htpasswd -i /etc/st2/htpasswd $USERNAME +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" +step "Install key/value store (Redis)" +redis_install -install_st2web() { - # Add key and repo for the latest stable nginx - sudo rpm --import http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/yum.repos.d/nginx.repo -[nginx] -name=nginx repo -baseurl=http://nginx.org/packages/rhel/8/x86_64/ -gpgcheck=1 -enabled=1 -EOT" - - # Ensure that EPEL repo is not used for nginx - sudo sed -i 's/^\(enabled=1\)$/exclude=nginx\n\1/g' /etc/yum.repos.d/epel.repo - - # Install nginx - sudo yum install -y nginx - - # Install st2web - sudo yum install -y ${ST2WEB_PKG} - - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information Technology/CN=$(hostname)" - - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - - # EL8: Comment out server { block } in nginx.conf and clean up - # nginx 1.6 in EL8 ships with a server block enabled which needs to be disabled - - # back up conf - sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - # comment out server block eg. server {...} - sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak > /tmp/nginx.conf - # copy modified file over - sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf - # remove double comments - sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf - # remove comment closing out server block - sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf - - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - - sudo systemctl restart nginx - sudo systemctl enable nginx - - # RHEL 8 runs firewalld so we need to open http/https - if is_rhel && command -v firewall-cmd >/dev/null 2>&1; then - sudo firewall-cmd --zone=public --add-service=http --add-service=https - sudo firewall-cmd --zone=public --permanent --add-service=http --add-service=https - fi -} - -install_st2chatops() { - # Install st2chatops - sudo yum install -y ${ST2CHATOPS_PKG} -} - -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env - - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env - - sudo systemctl restart st2chatops - sudo systemctl enable st2chatops - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/rhel8.html#setup-chatops" - echo "########################################################" - fi -} +step "Install st2 (StackStorm)" +st2_install -trap 'fail' EXIT +step "Configure st2 system user account" +st2_configure_user + +step "Configure st2 authentication" +st2_configure_authentication + +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" + +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys + +step "Verify StackStorm installation" +st2_verification + +step "Install Web Interface (st2web)" +st2web_install + +step "Install ChatOps bot (st2chatops)" +st2chatops_install + +step "Configure st2chatops" +st2chatops_configure -STEP='Parse arguments' && setup_args $@ -STEP="Configure Proxy" && configure_proxy -STEP='Install net-tools' && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP='Adjust SELinux policies' && adjust_selinux_policies -STEP='Install repoquery tool' && install_yum_utils -STEP="Generate random password" && generate_random_passwords - -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops trap - EXIT ok_message diff --git a/scripts/st2bootstrap-el8.template.sh b/scripts/st2bootstrap-el8.template.sh deleted file mode 100644 index d10e3720..00000000 --- a/scripts/st2bootstrap-el8.template.sh +++ /dev/null @@ -1,405 +0,0 @@ -set -eu - -HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} -VERSION='' -RELEASE='stable' -REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' -DEV_BUILD='' -USERNAME='' -PASSWORD='' -ST2_PKG='st2' -ST2WEB_PKG='st2web' -ST2CHATOPS_PKG='st2chatops' - -is_rhel() { - return $(cat /etc/os-release | grep 'ID="rhel"') -} - -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - *) - # unknown option - ;; - esac - done - - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - - if [[ "$VERSION" != '' ]]; then - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 - fi - - if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing st2 $RELEASE $VERSION " - echo "########################################################" - - if [[ "$REPO_TYPE" == "staging" ]]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [[ "$DEV_BUILD" != '' ]]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [[ "${PASSWORD}" = '' ]]; then - echo "Password cannot be empty." - exit 1 - fi - fi -} - - -# include:includes/common.sh -# include:includes/rhel.sh - - -# Note that default SELINUX policies for RHEL8 differ with Rocky8. Rocky8 is more permissive by default -# Note that depending on distro assembly/settings you may need more rules to change -# Apply these changes OR disable selinux in /etc/selinux/config (manually) -adjust_selinux_policies() { - if getenforce | grep -q 'Enforcing'; then - # SELINUX management tools, not available for some minimal installations - sudo yum install -y policycoreutils-python-utils - - # Allow rabbitmq to use '25672' port, otherwise it will fail to start - sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 - - # Allow network access for nginx - sudo setsebool -P httpd_can_network_connect 1 - fi -} - -install_net_tools() { - # Install netstat - sudo yum install -y net-tools -} - -install_st2_dependencies() { - # RabbitMQ on RHEL8 requires module(perl:5.26 - if is_rhel; then - sudo yum -y module enable perl:5.26 - fi - - is_epel_installed=$(rpm -qa | grep epel-release || true) - if [[ -z "$is_epel_installed" ]]; then - sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - fi - - # Various other dependencies needed by st2 and installer script - sudo yum -y install crudini -} - -install_rabbitmq() { - # Install erlang from rabbitmq/erlang as need newer version - # than available in epel. - curl -sL https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash - sudo yum -y install erlang-25* - # Install rabbit from packagecloud - curl -sL https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash - sudo yum makecache -y --disablerepo='*' --enablerepo='rabbitmq_rabbitmq-server' - - sudo yum -y install curl rabbitmq-server - - # Configure RabbitMQ to listen on localhost only - sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' - - sudo systemctl start rabbitmq-server - sudo systemctl enable rabbitmq-server - - sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" - sudo rabbitmqctl delete_user guest - sudo rabbitmqctl set_user_tags stackstorm administrator - sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" -} - -install_mongodb() { - - # Add key and repo for the latest stable MongoDB (4.0) - sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo -[mongodb-org-4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/4.0/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc -EOT" - - sudo yum -y install mongodb-org - - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf - - sudo systemctl start mongod - sudo systemctl enable mongod - - sleep 5 - - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <> /etc/mongod.conf' - - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod -} - -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo yum install -y redis - sudo systemctl start redis - sudo systemctl enable redis -} - -install_st2() { - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash - - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo yum -y install ${ST2_PKG} - else - sudo yum -y install jq - - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "el8" "st2-.*.rpm") - sudo yum -y install ${PACKAGE_URL} - fi - - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" - - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" - - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" - - sudo st2ctl start - sudo st2ctl reload --register-all -} - - -configure_st2_authentication() { - # Install htpasswd tool - sudo yum -y install httpd-tools - - # Create a user record in a password file. - echo $PASSWORD | sudo htpasswd -i /etc/st2/htpasswd $USERNAME - - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' - - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} - - -install_st2web() { - # Add key and repo for the latest stable nginx - sudo rpm --import http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/yum.repos.d/nginx.repo -[nginx] -name=nginx repo -baseurl=http://nginx.org/packages/rhel/8/x86_64/ -gpgcheck=1 -enabled=1 -EOT" - - # Ensure that EPEL repo is not used for nginx - sudo sed -i 's/^\(enabled=1\)$/exclude=nginx\n\1/g' /etc/yum.repos.d/epel.repo - - # Install nginx - sudo yum install -y nginx - - # Install st2web - sudo yum install -y ${ST2WEB_PKG} - - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information Technology/CN=$(hostname)" - - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - - # EL8: Comment out server { block } in nginx.conf and clean up - # nginx 1.6 in EL8 ships with a server block enabled which needs to be disabled - - # back up conf - sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - # comment out server block eg. server {...} - sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak > /tmp/nginx.conf - # copy modified file over - sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf - # remove double comments - sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf - # remove comment closing out server block - sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf - - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - - sudo systemctl restart nginx - sudo systemctl enable nginx - - # RHEL 8 runs firewalld so we need to open http/https - if is_rhel && command -v firewall-cmd >/dev/null 2>&1; then - sudo firewall-cmd --zone=public --add-service=http --add-service=https - sudo firewall-cmd --zone=public --permanent --add-service=http --add-service=https - fi -} - -install_st2chatops() { - # Install st2chatops - sudo yum install -y ${ST2CHATOPS_PKG} -} - -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env - - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env - - sudo systemctl restart st2chatops - sudo systemctl enable st2chatops - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/rhel8.html#setup-chatops" - echo "########################################################" - fi -} - -trap 'fail' EXIT - -STEP='Parse arguments' && setup_args $@ -STEP="Configure Proxy" && configure_proxy -STEP='Install net-tools' && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP='Adjust SELinux policies' && adjust_selinux_policies -STEP='Install repoquery tool' && install_yum_utils -STEP="Generate random password" && generate_random_passwords - -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops -trap - EXIT - -ok_message diff --git a/scripts/st2bootstrap-el9.sh b/scripts/st2bootstrap-el9.sh index 3269747d..2137e631 100644 --- a/scripts/st2bootstrap-el9.sh +++ b/scripts/st2bootstrap-el9.sh @@ -1,567 +1,904 @@ #!/usr/bin/env bash -# NOTE: This file is automatically generated by the tools/generate_final_installer_scripts.py -# script using the template file and common include files in scripts/includes/*.sh. # -# DO NOT EDIT MANUALLY. # -# Please edit corresponding template file and include files. -set -eu +set -e -u +x HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} VERSION='' RELEASE='stable' REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' DEV_BUILD='' USERNAME='' PASSWORD='' ST2_PKG='st2' ST2WEB_PKG='st2web' ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 -is_rhel() { - return $(cat /etc/os-release | grep 'ID="rhel"') -} +declare -A INSTALL_TYPE=() -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - *) - # unknown option - ;; - esac - done +source <(sed 's/^/OS_/g' /etc/os-release) + +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + + echo "${OUTPUT}" +} +function cecho() +{ + if [[ "$1" == "-n" ]]; then + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} +setup_install_parameters() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + if [[ -n "$DEV_BUILD" ]]; then + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - if [[ "$VERSION" != '' ]]; then +setup_check_version() +{ + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing st2 $RELEASE $VERSION " - echo "########################################################" - - if [[ "$REPO_TYPE" == "staging" ]]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [[ "$DEV_BUILD" != '' ]]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [[ "${PASSWORD}" = '' ]]; then - echo "Password cannot be empty." - exit 1 + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi fi - fi -} - - -function configure_proxy() { - # Allow bypassing 'proxy' env vars via sudo - local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' - if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" - fi - - # Configure proxy env vars for 'st2api', 'st2actionrunner' and 'st2chatops' system configs - # See: https://docs.stackstorm.com/packs.html#installing-packs-from-behind-a-proxy - local service_config_path=$(hash apt-get >/dev/null 2>&1 && echo '/etc/default' || echo '/etc/sysconfig') - for service in st2api st2actionrunner st2chatops; do - service_config="${service_config_path}/${service}" - # create file if doesn't exist yet - sudo test -e ${service_config} || sudo touch ${service_config} - for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do - # delete line from file if specific proxy env var is unset - if sudo test -z "${!env_var:-}"; then - sudo sed -i "/^${env_var}=/d" ${service_config} - # add proxy env var if it doesn't exist yet - elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then - sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" - # modify existing proxy env var value - elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then - sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} - fi - done - done + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" } -function get_package_url() { - # Retrieve direct package URL for the provided dev build, subtype and package name regex. - DEV_BUILD=$1 # Repo name and build number - / (e.g. st2/5646) - DISTRO=$2 # Distro name (e.g. focal,jammy,el8,el9) - PACKAGE_NAME_REGEX=$3 +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} +pkg_install() +{ + + sudo dnf -y install $@ +} - PACKAGES_METADATA=$(curl -sSL -q https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts) +pkg_meta_update() +{ + + sudo dnf -y check-update + +} + +pkg_is_installed() +{ + PKG="$1" + + sudo rpm -q "$PKG" | grep -qE "^${PKG}" + +} + + +pkg_get_latest_version() +{ + local PKG="$1" # st2 + local VERSION="$2" # 3.9dev + LATEST=$(repoquery -y --nvr --show-duplicates "$PKG" | grep -F "${PKG}-${VERSION}" | sort --version-sort | tail -n 1) + echo "${LATEST#*-}" +} - if [ -z "${PACKAGES_METADATA}" ]; then - echo "Failed to retrieve packages metadata from https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" 1>&2 - return 2 - fi - PACKAGES_URLS="$(echo ${PACKAGES_METADATA} | jq -r '.[].url')" - PACKAGE_URL=$(echo "${PACKAGES_URLS}" | egrep "${DISTRO}/${PACKAGE_NAME_REGEX}") +repo_add_gpg_key() +{ + KEY_NAME="$1" + KEY_URL="$2" + rpm --import "${KEY_URL}" +} + - if [ -z "${PACKAGE_URL}" ]; then - echo "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" 1>&2 - echo "Circle CI response: ${PACKAGES_METADATA}" 1>&2 - return 2 - fi +repo_definition() +{ + REPO_NAME="$1" + REPO_URL="$2" + KEY_NAME="$3" + KEY_URL="$4" + REPO_PATH="/etc/yum.repos.d/" - echo ${PACKAGE_URL} + cat <"${REPO_PATH}/${REPO_NAME}.repo" +[${REPO_NAME}] +name=${REPO_NAME} +baseurl=${REPO_URL} +repo_gpgcheck=1 +enabled=1 +gpgkey=${KEY_URL} +gpgcheck=0 +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +pkg_gpgcheck=1 +autorefresh=1 +type=rpm-md +EOF } -function port_status() { - # If the specified tcp4 port is bound, then return the "port pid/procname", - # else if a pipe command fails, return "Unbound", - # else return "". - # - # Please note that all return values end with a newline. - # - # Use netstat and awk to get a list of all the tcp4 sockets that are in the LISTEN state, - # matching the specified port. - # - # The `netstat -tunlp --inet` command is assumed to output data in the following format: - # Active Internet connections (only servers) - # Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name - # tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7506/httpd - # - # The awk command prints the 4th and 7th columns of any line matching both the following criteria: - # 1) The 4th column contains the port passed to port_status() (i.e., $1) - # 2) The 6th column contains "LISTEN" - # - # Sample output: - # 0.0.0.0:25000 7506/sshd - ret=$(sudo netstat -tunlp --inet | awk -v port=":$1$" '$4 ~ port && $6 ~ /LISTEN/ { print $4 " " $7 }' || echo 'Unbound'); - echo "$ret"; +pkg_get_versions() +{ + dnf -y info --showduplicates "$1" } -check_st2_host_dependencies() { - # CHECK 1: Determine which, if any, of the required ports are used by an existing process. +repo_clean_meta() +{ + dnf -y clean metadata + dnf -y clean dbcache + dnf -y clean all +} + - # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) - # and st2 (9100-9102). +repo_pkg_availability() { + local PKG="$1" + local VERSION="$2" - declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") - declare -a used=() + local PKG_VER="" + + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + - for i in "${ports[@]}" - do - rv=$(port_status $i | sed 's/.*-$\|.*systemd\|.*beam.smp.*\|.*epmd\|.*st2.*\|.*nginx.*\|.*python.*\|.*postmaster.*\|.*mongod\|.*init//') - if [ "$rv" != "Unbound" ] && [ "$rv" != "" ]; then - used+=("$rv") + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} +system_install_runtime_packages() +{ + + if ! pkg_is_installed epel-release; then + pkg_install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + fi + + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + yum-utils + iproute + gnupg2 + httpd-tools + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" fi - done - # If any used ports were found, display helpful message and exit - if [ ${#used[@]} -gt 0 ]; then - printf "\nNot all required TCP ports are available. ST2 and related services will fail to start.\n\n" - echo "The following ports are in use by the specified pid/process and need to be stopped:" - for port_pid_process in "${used[@]}" + service_config_path="" + for cfgdir in "/etc/sysconfig" "/etc/default" do - echo " $port_pid_process" + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi done - echo "" - exit 1 - fi - # CHECK 2: Ensure there is enough space at /var/lib/mongodb - VAR_SPACE=`df -Pk /var/lib | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{print $4}'` - if [ ${VAR_SPACE} -lt 358400 ]; then - echo "" - echo "MongoDB requires at least 350MB free in /var/lib/mongodb" - echo "There is not enough space for MongoDB. It will fail to start." - echo "Please, add some space to /var or clean it up." - exit 1 - fi + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done } -generate_random_passwords() { - # Generate random password used for MongoDB user authentication - ST2_MONGODB_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') - # Generate random password used for RabbitMQ user authentication - ST2_RABBITMQ_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24 ; echo '') +system_port_status() +{ + # + # + # + + # + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" } -configure_st2_user () { - # Create an SSH system user (default `stanley` user may be already created) - if (! id stanley 2>/dev/null); then - sudo useradd stanley - fi +system_check_resources() +{ + + PORT_TEST=$( + cat <> ${SYSTEM_HOME}/.ssh/authorized_keys" - fi +system_generate_password() +{ + local LEN="$1" + head /dev/urandom | tr -dc A-Za-z0-9 | head -c $LEN; echo '' +} - sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys - sudo chmod 0700 ${SYSTEM_HOME}/.ssh - sudo chown -R stanley:stanley ${SYSTEM_HOME} - # Enable passwordless sudo - local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" - if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" - fi +ok_message() +{ + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} - ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" - ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo +} - CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" - CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" - if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then - sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} - fi +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} - sudo sh -c "cat < ${ROOT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/el/9/\$basearch/" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + echo "el9" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + pkg_install "${PACKAGE_URL}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}-${VERSION}" +} - # Write config for root user - if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then - return - fi - # Write config for current user (in case current user != root) - if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then - sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} - fi +st2_install_dev_build() +{ + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" - sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} -[credentials] -username = ${USERNAME} -password = ${PASSWORD} -EOT" + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi - # Fix the permissions - sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" } +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="/etc/st2/st2.conf" -generate_symmetric_crypto_key_for_datastore() { - DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" - DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" - sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" - # If the file ${DATASTORE_ENCRYPTION_KEY_PATH} exists and is not empty, then do not generate - # a new key. st2-generate-symmetric-crypto-key fails if the key file already exists. - if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then - sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} - fi + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} - # Make sure only st2 user can read the file - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} - sudo chgrp st2 ${DATASTORE_ENCRYPTION_KEY_PATH} - sudo chmod o-r ${DATASTORE_ENCRYPTION_KEY_PATH} - # set path to the key file in the config - sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} +st2_configure_authentication() { + local ST2_CFGFILE="/etc/st2/st2.conf" - # NOTE: We need to restart all the affected services so they pick the key and load it in memory - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2sensorcontainer - sudo st2ctl restart-component st2workflowengine - sudo st2ctl restart-component st2actionrunner + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done } -verify_st2() { - st2 --version - st2 -h +st2_configure_user() +{ + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi - st2 auth $USERNAME -p $PASSWORD - # A shortcut to authenticate and export the token - export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) + SYSTEM_HOME=$(echo ~stanley) - # List the actions from a 'core' pack - st2 action list --pack=core + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi - # Run a local shell command - st2 run core.local -- date -R + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi - # See the execution results - st2 execution list + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi - # Fire a remote comand via SSH (Requires passwordless SSH) - st2 run core.remote hosts='127.0.0.1' -- uname -a + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} - # Install a pack - st2 pack install st2 -} + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + sudo chmod 0440 /etc/sudoers.d/st2 -ok_message() { - echo "" - echo "" - echo "███████╗████████╗██████╗ ██████╗ ██╗ ██╗"; - echo "██╔════╝╚══██╔══╝╚════██╗ ██╔═══██╗██║ ██╔╝"; - echo "███████╗ ██║ █████╔╝ ██║ ██║█████╔╝ "; - echo "╚════██║ ██║ ██╔═══╝ ██║ ██║██╔═██╗ "; - echo "███████║ ██║ ███████╗ ╚██████╔╝██║ ██╗"; - echo "╚══════╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═╝"; - echo "" - echo " st2 is installed and ready to use." - echo "" - echo "Head to https://YOUR_HOST_IP/ to access the WebUI" - echo "" - echo "Don't forget to dive into our documentation! Here are some resources" - echo "for you:" - echo "" - echo "* Documentation - https://docs.stackstorm.com" - echo "* Pack Exchange - https://exchange.stackstorm.org/" - echo "" - echo "Thanks for installing StackStorm! Come visit us in our Slack Channel" - echo "and tell us how it's going. We'd love to hear from you!" - echo "http://stackstorm.com/community-signup" + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers } -fail() { - echo "############### ERROR ###############" - echo "# Failed on $STEP #" - echo "#####################################" - exit 2 -} +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) -install_yum_utils() { - # We need repoquery tool to get package_name-package_ver-package_rev in RPM based distros - # if we don't want to construct this string manually using yum info --show-duplicates and - # doing a bunch of sed awk magic. Problem is this is not installed by default on all images. - sudo yum install -y yum-utils -} + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" -get_full_pkg_versions() { - if [ "$VERSION" != '' ]; - then - local RHMAJVER=`cat /etc/redhat-release | sed 's/[^0-9.]*\([0-9.]\).*/\1/'` - local YES_FLAG="" - if [ "$RHMAJVER" -ge "8" ]; then - # RHEL 8 and newer, you need "-y" flag to avoid being prompted to confirm "yes" - local YES_FLAG="-y" + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} fi - local ST2_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2 | grep -F st2-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2_VER" ]; then - echo "Could not find requested version of st2!!!" - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2 - exit 3 + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return fi - ST2_PKG=${ST2_VER} - local ST2WEB_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2web | grep -F st2web-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2WEB_VER" ]; then - echo "Could not find requested version of st2web." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2web - exit 3 + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} fi - ST2WEB_PKG=${ST2WEB_VER} - local ST2CHATOPS_VER=$(repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops | grep -F st2chatops-${VERSION} | sort --version-sort | tail -n 1) - if [ -z "$ST2CHATOPS_VER" ]; then - echo "Could not find requested version of st2chatops." - sudo repoquery ${YES_FLAG} --nvr --show-duplicates st2chatops - exit 3 + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} +} + + +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} fi - ST2CHATOPS_PKG=${ST2CHATOPS_VER} - echo "##########################################################" - echo "#### Following versions of packages will be installed ####" - echo "${ST2_PKG}" - echo "${ST2WEB_PKG}" - echo "${ST2CHATOPS_PKG}" - echo "##########################################################" - fi + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done } +st2_verification() +{ + echo.info "Check version" + st2 --version + + echo.info "Check help" + st2 -h + + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) -# Note that default SELINUX policies for RHEL9 differ with CentOS8. CentOS8 is more permissive by default -# Note that depending on distro assembly/settings you may need more rules to change -# Apply these changes OR disable selinux in /etc/selinux/config (manually) -adjust_selinux_policies() { - if getenforce | grep -q 'Enforcing'; then - # SELINUX management tools, not available for some minimal installations - sudo yum install -y policycoreutils-python-utils + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core - # Allow rabbitmq to use '25672' port, otherwise it will fail to start - sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 + echo.info "Check local shell command" + st2 run core.local -- date -R - # Allow network access for nginx - sudo setsebool -P httpd_can_network_connect 1 - fi -} - -install_net_tools() { - # Install netstat - sudo yum install -y net-tools -} - -install_st2_dependencies() { - # RabbitMQ on RHEL9 requires module(perl:5.26 - if is_rhel; then - sudo yum -y module enable perl:5.26 - fi + echo.info "Check execution list" + st2 execution list - is_epel_installed=$(rpm -qa | grep epel-release || true) - if [[ -z "$is_epel_installed" ]]; then - sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - fi + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a - # Various other dependencies needed by st2 and installer script - sudo yum -y install crudini + echo.info "Check pack installation" + st2 pack install st2 +} +nodejs_configure_repository() +{ + curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo -E bash - } -install_rabbitmq() { - # Install erlang from rabbitmq/erlang as need newer version - # than available in epel. - curl -sL https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash - sudo yum -y install erlang-25* - # Install rabbit from packagecloud - curl -sL https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash - sudo yum makecache -y --disablerepo='*' --enablerepo='rabbitmq_rabbitmq-server' - - sudo yum -y install curl rabbitmq-server +st2chatops_install() +{ + nodejs_configure_repository + pkg_install nodejs - # Configure RabbitMQ to listen on localhost only - sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} +} - sudo systemctl start rabbitmq-server - sudo systemctl enable rabbitmq-server +st2chatops_configure() +{ + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/rhel/9/x86_64/" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" - sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" - sudo rabbitmqctl delete_user guest - sudo rabbitmqctl set_user_tags stackstorm administrator - sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" } -install_mongodb() { +st2web_install() +{ + nginx_configure_repo + pkg_meta_update - # Add key and repo for the latest stable MongoDB (4.0) - sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo -[mongodb-org-4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/4.0/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc -EOT" + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} + + sudo mkdir -p /etc/ssl/st2 + sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ + -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ + Technology/CN=$(hostname)" - sudo yum -y install mongodb-org + sudo rm -f /etc/nginx/conf.d/default.conf - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf + sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak + sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak >/etc/nginx/nginx.conf + sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf + sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf + - sudo systemctl start mongod - sudo systemctl enable mongod + sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - sleep 5 + sudo systemctl enable nginx + sudo systemctl restart nginx +} +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/yum/redhat/9/mongodb-org/7.0/x86_64/" \ + "mongodb-org-7.0-key" \ + "https://pgp.mongodb.com/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongod" + local DB_PATH="/var/lib/mongo" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" + mongodb_adjust_selinux_policies +} - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <${CFGFILE}" +} + +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} + +mongodb_install() +{ + local MONGODB_PKG=mongodb-org + + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi + + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration + + sudo systemctl enable mongod + sudo systemctl start mongod + + sleep 10 + + mongosh <> /etc/mongod.conf' + sudo sed -ri 's/^ authorization: disabled$/ authorization: enabled/g' /etc/mongod.conf - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod + sudo systemctl restart mongod } +rabbitmq_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + pkg_install policycoreutils-python-utils + + sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo yum install -y redis - sudo systemctl start redis - sudo systemctl enable redis + sudo setsebool -P httpd_can_network_connect 1 + fi } -install_st2() { - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash +rabbitmq_install() +{ + local RABBITMQ_PKG=rabbitmq-server - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo yum -y install ${ST2_PKG} - else - sudo yum -y install jq + if [[ $INSTALL_RABBITMQ -eq 0 ]]; then + echo.info "Skip RabbitMQ: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$RABBITMQ_PKG"; then + echo.info "Skip RabbitMQ: Package is already present on the system." + return + fi + repo_definition "erlang" \ + "https://yum1.rabbitmq.com/erlang/el/9/\$basearch" \ + "erlang-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key" + repo_definition "rabbitmq-server" \ + "https://yum2.rabbitmq.com/rabbitmq/el/9/\$basearch" \ + "rabbitmq-server-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc" + repo_definition "rabbitmq-server-noarch" \ + "https://yum2.rabbitmq.com/rabbitmq/el/9/noarch" \ + "rabbitmq-server-key" \ + "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key https://github.com/rabbitmq/signing-keys/releases/download/3.0/rabbitmq-release-signing-key.asc" + + rabbitmq_adjust_selinux_policies + + local PKGS=( + erlang + "$RABBITMQ_PKG" + ) + + + pkg_meta_update + pkg_install ${PKGS[@]} + + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server + + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi + local REDIS_SERVICE=redis + + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "el9" "st2-.*.rpm") - sudo yum -y install ${PACKAGE_URL} - fi + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" +} - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" +trap 'fail' EXIT - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" +step "Setup runtime arguments" +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password - sudo st2ctl start - sudo st2ctl reload --register-all -} +step "Install required runtime packages" +system_install_runtime_packages +step "Check storage capacity and network ports" +system_check_resources -configure_st2_authentication() { - # Install htpasswd tool - sudo yum -y install httpd-tools +step "Configure HTTP Proxy" +system_configure_proxy - # Create a user record in a password file. - echo $PASSWORD | sudo htpasswd -i /etc/st2/htpasswd $USERNAME +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" +step "Install key/value store (Redis)" +redis_install -install_st2web() { - # Add key and repo for the latest stable nginx - sudo rpm --import http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/yum.repos.d/nginx.repo -[nginx] -name=nginx repo -baseurl=http://nginx.org/packages/rhel/8/x86_64/ -gpgcheck=1 -enabled=1 -EOT" - - # Ensure that EPEL repo is not used for nginx - sudo sed -i 's/^\(enabled=1\)$/exclude=nginx\n\1/g' /etc/yum.repos.d/epel.repo - - # Install nginx - sudo yum install -y nginx - - # Install st2web - sudo yum install -y ${ST2WEB_PKG} - - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information Technology/CN=$(hostname)" - - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - - # EL9: Comment out server { block } in nginx.conf and clean up - # nginx 1.6 in EL9 ships with a server block enabled which needs to be disabled - - # back up conf - sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - # comment out server block eg. server {...} - sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak > /tmp/nginx.conf - # copy modified file over - sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf - # remove double comments - sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf - # remove comment closing out server block - sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf - - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - - sudo systemctl restart nginx - sudo systemctl enable nginx - - # RHEL 9 runs firewalld so we need to open http/https - if is_rhel && command -v firewall-cmd >/dev/null 2>&1; then - sudo firewall-cmd --zone=public --add-service=http --add-service=https - sudo firewall-cmd --zone=public --permanent --add-service=http --add-service=https - fi -} - -install_st2chatops() { - # Install st2chatops - sudo yum install -y ${ST2CHATOPS_PKG} -} - -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env - - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env - - sudo systemctl restart st2chatops - sudo systemctl enable st2chatops - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/rhel9.html#setup-chatops" - echo "########################################################" - fi -} +step "Install st2 (StackStorm)" +st2_install -trap 'fail' EXIT +step "Configure st2 system user account" +st2_configure_user + +step "Configure st2 authentication" +st2_configure_authentication + +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" + +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys + +step "Verify StackStorm installation" +st2_verification + +step "Install Web Interface (st2web)" +st2web_install + +step "Install ChatOps bot (st2chatops)" +st2chatops_install + +step "Configure st2chatops" +st2chatops_configure -STEP='Parse arguments' && setup_args $@ -STEP="Configure Proxy" && configure_proxy -STEP='Install net-tools' && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP='Adjust SELinux policies' && adjust_selinux_policies -STEP='Install repoquery tool' && install_yum_utils -STEP="Generate random password" && generate_random_passwords - -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops trap - EXIT ok_message diff --git a/scripts/st2bootstrap-el9.template.sh b/scripts/st2bootstrap-el9.template.sh deleted file mode 100644 index e922fa62..00000000 --- a/scripts/st2bootstrap-el9.template.sh +++ /dev/null @@ -1,405 +0,0 @@ -set -eu - -HUBOT_ADAPTER='slack' -HUBOT_SLACK_TOKEN=${HUBOT_SLACK_TOKEN:-''} -VERSION='' -RELEASE='stable' -REPO_TYPE='' -REPO_PREFIX='' -ST2_PKG_VERSION='' -DEV_BUILD='' -USERNAME='' -PASSWORD='' -ST2_PKG='st2' -ST2WEB_PKG='st2web' -ST2CHATOPS_PKG='st2chatops' - -is_rhel() { - return $(cat /etc/os-release | grep 'ID="rhel"') -} - -setup_args() { - for i in "$@" - do - case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -s|--stable) - RELEASE=stable - shift - ;; - -u|--unstable) - RELEASE=unstable - shift - ;; - --staging) - REPO_TYPE='staging' - shift - ;; - --dev=*) - DEV_BUILD="${i#*=}" - shift - ;; - --user=*) - USERNAME="${i#*=}" - shift - ;; - --password=*) - PASSWORD="${i#*=}" - shift - ;; - *) - # unknown option - ;; - esac - done - - if [[ "$REPO_TYPE" != '' ]]; then - REPO_PREFIX="${REPO_TYPE}-" - fi - - if [[ "$VERSION" != '' ]]; then - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "$VERSION does not match supported formats x.y.z or x.ydev" - exit 1 - fi - - if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then - echo "You're requesting a dev version! Switching to unstable!" - RELEASE='unstable' - fi - fi - - echo "########################################################" - echo " Installing st2 $RELEASE $VERSION " - echo "########################################################" - - if [[ "$REPO_TYPE" == "staging" ]]; then - printf "\n\n" - echo "################################################################" - echo "### Installing from staging repos!!! USE AT YOUR OWN RISK!!! ###" - echo "################################################################" - fi - - if [[ "$DEV_BUILD" != '' ]]; then - printf "\n\n" - echo "###############################################################################" - echo "### Installing from dev build artifacts!!! REALLY, ANYTHING COULD HAPPEN!!! ###" - echo "###############################################################################" - fi - - if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then - echo "Let's set StackStorm admin credentials." - echo "You can also use \"--user\" and \"--password\" for unattended installation." - echo "Press \"ENTER\" to continue or \"CTRL+C\" to exit/abort" - read -e -p "Admin username: " -i "st2admin" USERNAME - read -e -s -p "Password: " PASSWORD - - if [[ "${PASSWORD}" = '' ]]; then - echo "Password cannot be empty." - exit 1 - fi - fi -} - - -# include:includes/common.sh -# include:includes/rhel.sh - - -# Note that default SELINUX policies for RHEL9 differ with CentOS8. CentOS8 is more permissive by default -# Note that depending on distro assembly/settings you may need more rules to change -# Apply these changes OR disable selinux in /etc/selinux/config (manually) -adjust_selinux_policies() { - if getenforce | grep -q 'Enforcing'; then - # SELINUX management tools, not available for some minimal installations - sudo yum install -y policycoreutils-python-utils - - # Allow rabbitmq to use '25672' port, otherwise it will fail to start - sudo semanage port --list | grep -q 25672 || sudo semanage port -a -t amqp_port_t -p tcp 25672 - - # Allow network access for nginx - sudo setsebool -P httpd_can_network_connect 1 - fi -} - -install_net_tools() { - # Install netstat - sudo yum install -y net-tools -} - -install_st2_dependencies() { - # RabbitMQ on RHEL9 requires module(perl:5.26 - if is_rhel; then - sudo yum -y module enable perl:5.26 - fi - - is_epel_installed=$(rpm -qa | grep epel-release || true) - if [[ -z "$is_epel_installed" ]]; then - sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - fi - - # Various other dependencies needed by st2 and installer script - sudo yum -y install crudini -} - -install_rabbitmq() { - # Install erlang from rabbitmq/erlang as need newer version - # than available in epel. - curl -sL https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash - sudo yum -y install erlang-25* - # Install rabbit from packagecloud - curl -sL https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash - sudo yum makecache -y --disablerepo='*' --enablerepo='rabbitmq_rabbitmq-server' - - sudo yum -y install curl rabbitmq-server - - # Configure RabbitMQ to listen on localhost only - sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' - - sudo systemctl start rabbitmq-server - sudo systemctl enable rabbitmq-server - - sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" - sudo rabbitmqctl delete_user guest - sudo rabbitmqctl set_user_tags stackstorm administrator - sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" -} - -install_mongodb() { - - # Add key and repo for the latest stable MongoDB (4.0) - sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo -[mongodb-org-4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/4.0/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc -EOT" - - sudo yum -y install mongodb-org - - # Configure MongoDB to listen on localhost only - sudo sed -i -e "s#bindIp:.*#bindIp: 127.0.0.1#g" /etc/mongod.conf - - sudo systemctl start mongod - sudo systemctl enable mongod - - sleep 5 - - # Create admin user and user used by StackStorm (MongoDB needs to be running) - # NOTE: mongo shell will automatically exit when piping from stdin. There is - # no need to put quit(); at the end. This way last command exit code will be - # correctly preserved and install script will correctly fail and abort if this - # command fails. - mongo <> /etc/mongod.conf' - - # MongoDB needs to be restarted after enabling auth - sudo systemctl restart mongod -} - -install_redis() { - # Install Redis Server. By default, redis only listen on localhost only. - sudo yum install -y redis - sudo systemctl start redis - sudo systemctl enable redis -} - -install_st2() { - curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash - - if [[ "$DEV_BUILD" = '' ]]; then - STEP="Get package versions" && get_full_pkg_versions && STEP="Install st2" - sudo yum -y install ${ST2_PKG} - else - sudo yum -y install jq - - PACKAGE_URL=$(get_package_url "${DEV_BUILD}" "el9" "st2-.*.rpm") - sudo yum -y install ${PACKAGE_URL} - fi - - # Configure [database] section in st2.conf (username password for MongoDB access) - sudo crudini --set /etc/st2/st2.conf database username "stackstorm" - sudo crudini --set /etc/st2/st2.conf database password "${ST2_MONGODB_PASSWORD}" - - # Configure [messaging] section in st2.conf (username password for RabbitMQ access) - AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" - sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" - - # Configure [coordination] section in st2.conf (url for Redis access) - sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" - - sudo st2ctl start - sudo st2ctl reload --register-all -} - - -configure_st2_authentication() { - # Install htpasswd tool - sudo yum -y install httpd-tools - - # Create a user record in a password file. - echo $PASSWORD | sudo htpasswd -i /etc/st2/htpasswd $USERNAME - - # Configure [auth] section in st2.conf - sudo crudini --set /etc/st2/st2.conf auth enable 'True' - sudo crudini --set /etc/st2/st2.conf auth backend 'flat_file' - sudo crudini --set /etc/st2/st2.conf auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' - - sudo st2ctl restart-component st2auth - sudo st2ctl restart-component st2api - sudo st2ctl restart-component st2stream -} - - -install_st2web() { - # Add key and repo for the latest stable nginx - sudo rpm --import http://nginx.org/keys/nginx_signing.key - sudo sh -c "cat < /etc/yum.repos.d/nginx.repo -[nginx] -name=nginx repo -baseurl=http://nginx.org/packages/rhel/8/x86_64/ -gpgcheck=1 -enabled=1 -EOT" - - # Ensure that EPEL repo is not used for nginx - sudo sed -i 's/^\(enabled=1\)$/exclude=nginx\n\1/g' /etc/yum.repos.d/epel.repo - - # Install nginx - sudo yum install -y nginx - - # Install st2web - sudo yum install -y ${ST2WEB_PKG} - - # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 - sudo mkdir -p /etc/ssl/st2 - - sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ - -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information Technology/CN=$(hostname)" - - # Remove default site, if present - sudo rm -f /etc/nginx/conf.d/default.conf - - # EL9: Comment out server { block } in nginx.conf and clean up - # nginx 1.6 in EL9 ships with a server block enabled which needs to be disabled - - # back up conf - sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - # comment out server block eg. server {...} - sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak > /tmp/nginx.conf - # copy modified file over - sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf - # remove double comments - sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf - # remove comment closing out server block - sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf - - # Copy and enable StackStorm's supplied config file - sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ - - sudo systemctl restart nginx - sudo systemctl enable nginx - - # RHEL 9 runs firewalld so we need to open http/https - if is_rhel && command -v firewall-cmd >/dev/null 2>&1; then - sudo firewall-cmd --zone=public --add-service=http --add-service=https - sudo firewall-cmd --zone=public --permanent --add-service=http --add-service=https - fi -} - -install_st2chatops() { - # Install st2chatops - sudo yum install -y ${ST2CHATOPS_PKG} -} - -configure_st2chatops() { - # set API keys. This should work since CLI is configuered already. - ST2_API_KEY=`st2 apikey create -k` - sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env - - sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env - - # Setup adapter - if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_TOKEN" ]] - then - sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^# (export HUBOT_SLACK_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env - sudo sed -i -r "s/^(export HUBOT_SLACK_TOKEN.).*/\1$HUBOT_SLACK_TOKEN/" /opt/stackstorm/chatops/st2chatops.env - - sudo systemctl restart st2chatops - sudo systemctl enable st2chatops - else - echo "####################### WARNING ########################" - echo "######## Chatops requires manual configuration #########" - echo "Edit /opt/stackstorm/chatops/st2chatops.env to specify " - echo "the adapter and settings hubot should use to connect to " - echo "the chat you're using. Don't forget to start the service" - echo "afterwards:" - echo "" - echo " $ sudo service st2chatops restart" - echo "" - echo "For more information, please refer to documentation at " - echo "https://docs.stackstorm.com/install/rhel9.html#setup-chatops" - echo "########################################################" - fi -} - -trap 'fail' EXIT - -STEP='Parse arguments' && setup_args $@ -STEP="Configure Proxy" && configure_proxy -STEP='Install net-tools' && install_net_tools -STEP="Check TCP ports and MongoDB storage requirements" && check_st2_host_dependencies -STEP='Adjust SELinux policies' && adjust_selinux_policies -STEP='Install repoquery tool' && install_yum_utils -STEP="Generate random password" && generate_random_passwords - -STEP="Install st2 dependencies" && install_st2_dependencies -STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq -STEP="Install st2 dependencies (MongoDB)" && install_mongodb -STEP="Install st2 dependencies (Redis)" && install_redis -STEP="Install st2" && install_st2 -STEP="Configure st2 user" && configure_st2_user -STEP="Configure st2 auth" && configure_st2_authentication -STEP="Configure st2 CLI config" && configure_st2_cli_config -STEP="Generate symmetric crypto key for datastore" && generate_symmetric_crypto_key_for_datastore -STEP="Verify st2" && verify_st2 - - -STEP="Install st2web" && install_st2web -STEP="Install st2chatops" && install_st2chatops -STEP="Configure st2chatops" && configure_st2chatops -trap - EXIT - -ok_message diff --git a/scripts/st2bootstrap-focal.sh b/scripts/st2bootstrap-focal.sh new file mode 100644 index 00000000..8e4e76b2 --- /dev/null +++ b/scripts/st2bootstrap-focal.sh @@ -0,0 +1,1187 @@ +#!/usr/bin/env bash +# +# + +set -e -u +x + +HUBOT_ADAPTER='slack' +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} +VERSION='' +RELEASE='stable' +REPO_TYPE='' +DEV_BUILD='' +USERNAME='' +PASSWORD='' +ST2_PKG='st2' +ST2WEB_PKG='st2web' +ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 + +declare -A INSTALL_TYPE=() + +source <(sed 's/^/OS_/g' /etc/os-release) + +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + + echo "${OUTPUT}" +} +function cecho() +{ + if [[ "$1" == "-n" ]]; then + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} +setup_install_parameters() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + if [[ -n "$DEV_BUILD" ]]; then + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} + + +setup_check_version() +{ + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 + fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" + + if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi + fi + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" +} + + +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} +pkg_install() +{ + + export DEBIAN_FRONTEND=noninteractive + + sudo apt -y install $@ +} + +pkg_meta_update() +{ + + sudo apt update -y + +} + +pkg_is_installed() +{ + PKG="$1" + + dpkg -l "$PKG" | grep -qE "^ii.*${PKG}" + +} + + + +repo_definition() +{ + REPO_PATH="/etc/apt/sources.list.d" + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + + REPO_NAME="$1" + REPO_URL="$2" + REPO_SUITES="$3" + REPO_COMPONENT="$4" + KEY_NAME="$5" + KEY_URL="$6" + + repo_add_gpg_key "$KEY_NAME" "$KEY_URL" + sudo cat <"${REPO_PATH}/${REPO_NAME}.sources" +Types: deb +URIs: ${REPO_URL} +Suites: ${REPO_SUITES} +Components: ${REPO_COMPONENT} +Architectures: $(dpkg --print-architecture) +Signed-By: ${GPG_KEY_PATH}/${KEY_NAME}.gpg +EOF +} + + +repo_add_gpg_key() +{ + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + KEY_NAME="$1" + KEY_URL="$2" + + curl -1sLf "$KEY_URL" | sudo gpg --dearmor -o "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" + sudo mv "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" "${GPG_KEY_PATH}/${KEY_NAME}.gpg" +} + + +pkg_get_latest_version() +{ + local PKG="$1" + local VERSION="$2" + apt-cache show "$PKG" | awk '/Version:/{print $2}' | grep "^${VERSION//./\\.}" | sort --version-sort | tail -n 1 +} + + +pkg_get_versions() +{ + apt-cache show "$1" +} + + +repo_clean_meta() +{ + true +} + + +repo_pkg_availability() { + local PKG="$1" + local VERSION="$2" + + local PKG_VER="" + + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + + + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} +system_install_runtime_packages() +{ + + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + + iproute2 + gnupg + apt-transport-https + apache2-utils + ca-certificates + + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" + fi + + service_config_path="" + for cfgdir in "/etc/sysconfig" "/etc/default" + do + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi + done + + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done +} + + +system_port_status() +{ + # + # + # + + # + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" +} + + +system_check_resources() +{ + + PORT_TEST=$( + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} + +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo +} + + +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} + + +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + echo "${OS_VERSION_CODENAME}" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + local PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" + curl -sSL -k -o "${PACKAGE_FILENAME}" "${PACKAGE_URL}" + sudo dpkg --install --force-depends "${PACKAGE_FILENAME}" + sudo apt install --yes --fix-broken + rm "${PACKAGE_FILENAME}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}=${VERSION}" +} + + +st2_install_dev_build() +{ + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" + + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi + + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" +} + +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="/etc/st2/st2.conf" + + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" + + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" + + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} + + +st2_configure_authentication() { + local ST2_CFGFILE="/etc/st2/st2.conf" + + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done +} + + +st2_configure_user() +{ + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi + + SYSTEM_HOME=$(echo ~stanley) + + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi + + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi + + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi + + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} + + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + + sudo chmod 0440 /etc/sudoers.d/st2 + + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers +} + + +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) + + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" + + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return + fi + + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} +} + + +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} + fi + + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done +} + + +st2_verification() +{ + echo.info "Check version" + st2 --version + + echo.info "Check help" + st2 -h + + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) + + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core + + echo.info "Check local shell command" + st2 run core.local -- date -R + + echo.info "Check execution list" + st2 execution list + + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a + + echo.info "Check pack installation" + st2 pack install st2 +} + +nodejs_configure_repository() +{ + curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - +} + +st2chatops_install() +{ + nodejs_configure_repository + pkg_install nodejs + + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} +} + +st2chatops_configure() +{ + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} + +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "nginx" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" +} + +st2web_install() +{ + nginx_configure_repo + pkg_meta_update + + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} + + sudo mkdir -p /etc/ssl/st2 + sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ + -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ + Technology/CN=$(hostname)" + + sudo rm -f /etc/nginx/conf.d/default.conf + + + + sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ + + sudo systemctl enable nginx + sudo systemctl restart nginx +} + + +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/apt/${OS_ID}" \ + "${OS_VERSION_CODENAME}/mongodb-org/7.0" \ + "multiverse" \ + "mongodb-org-7.0-key" \ + "https://www.mongodb.org/static/pgp/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongodb" + local DB_PATH="/var/lib/mongodb" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" +} + + +mongodb_write_configuration() +{ + local MONGODB_USER="$1" + local DB_PATH="$2" + local LOG_PATH="$3" + local CFGFILE="/etc/mongod.conf" + + TMP=$(cat <${CFGFILE}" +} + +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} + +mongodb_install() +{ + local MONGODB_PKG=mongodb-org + + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi + + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration + + sudo systemctl enable mongod + sudo systemctl start mongod + + sleep 10 + + mongosh </etc/apt/preferences.d/erlang.pref +Package: erlang* +Pin: origin ppa1.rabbitmq.com +Pin-Priority: 1001 +EOF + + + pkg_meta_update + pkg_install ${PKGS[@]} + + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server + + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi + + repo_definition "redis" \ + "https://packages.redis.io/deb" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "redis-key" \ + "https://packages.redis.io/gpg" + local REDIS_SERVICE=redis-server + + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi + + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" +} + +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done + +trap 'fail' EXIT + +step "Setup runtime arguments" +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password + +step "Install required runtime packages" +system_install_runtime_packages + +step "Check storage capacity and network ports" +system_check_resources + +step "Configure HTTP Proxy" +system_configure_proxy + +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords + +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" + +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" + +step "Install key/value store (Redis)" +redis_install + +step "Install st2 (StackStorm)" +st2_install + +step "Configure st2 system user account" +st2_configure_user + +step "Configure st2 authentication" +st2_configure_authentication + +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" + +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys + +step "Verify StackStorm installation" +st2_verification + +step "Install Web Interface (st2web)" +st2web_install + +step "Install ChatOps bot (st2chatops)" +st2chatops_install + +step "Configure st2chatops" +st2chatops_configure + +trap - EXIT + +ok_message diff --git a/scripts/st2bootstrap-jammy.sh b/scripts/st2bootstrap-jammy.sh new file mode 100644 index 00000000..8e4e76b2 --- /dev/null +++ b/scripts/st2bootstrap-jammy.sh @@ -0,0 +1,1187 @@ +#!/usr/bin/env bash +# +# + +set -e -u +x + +HUBOT_ADAPTER='slack' +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} +VERSION='' +RELEASE='stable' +REPO_TYPE='' +DEV_BUILD='' +USERNAME='' +PASSWORD='' +ST2_PKG='st2' +ST2WEB_PKG='st2web' +ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 + +declare -A INSTALL_TYPE=() + +source <(sed 's/^/OS_/g' /etc/os-release) + +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + + echo "${OUTPUT}" +} +function cecho() +{ + if [[ "$1" == "-n" ]]; then + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} +setup_install_parameters() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + if [[ -n "$DEV_BUILD" ]]; then + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} + + +setup_check_version() +{ + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 + fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" + + if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi + fi + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" +} + + +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} +pkg_install() +{ + + export DEBIAN_FRONTEND=noninteractive + + sudo apt -y install $@ +} + +pkg_meta_update() +{ + + sudo apt update -y + +} + +pkg_is_installed() +{ + PKG="$1" + + dpkg -l "$PKG" | grep -qE "^ii.*${PKG}" + +} + + + +repo_definition() +{ + REPO_PATH="/etc/apt/sources.list.d" + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + + REPO_NAME="$1" + REPO_URL="$2" + REPO_SUITES="$3" + REPO_COMPONENT="$4" + KEY_NAME="$5" + KEY_URL="$6" + + repo_add_gpg_key "$KEY_NAME" "$KEY_URL" + sudo cat <"${REPO_PATH}/${REPO_NAME}.sources" +Types: deb +URIs: ${REPO_URL} +Suites: ${REPO_SUITES} +Components: ${REPO_COMPONENT} +Architectures: $(dpkg --print-architecture) +Signed-By: ${GPG_KEY_PATH}/${KEY_NAME}.gpg +EOF +} + + +repo_add_gpg_key() +{ + GPG_KEY_PATH="/etc/apt/trusted.gpg.d" + KEY_NAME="$1" + KEY_URL="$2" + + curl -1sLf "$KEY_URL" | sudo gpg --dearmor -o "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" + sudo mv "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" "${GPG_KEY_PATH}/${KEY_NAME}.gpg" +} + + +pkg_get_latest_version() +{ + local PKG="$1" + local VERSION="$2" + apt-cache show "$PKG" | awk '/Version:/{print $2}' | grep "^${VERSION//./\\.}" | sort --version-sort | tail -n 1 +} + + +pkg_get_versions() +{ + apt-cache show "$1" +} + + +repo_clean_meta() +{ + true +} + + +repo_pkg_availability() { + local PKG="$1" + local VERSION="$2" + + local PKG_VER="" + + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + + + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} +system_install_runtime_packages() +{ + + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + + iproute2 + gnupg + apt-transport-https + apache2-utils + ca-certificates + + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" + fi + + service_config_path="" + for cfgdir in "/etc/sysconfig" "/etc/default" + do + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi + done + + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done +} + + +system_port_status() +{ + # + # + # + + # + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" +} + + +system_check_resources() +{ + + PORT_TEST=$( + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} + +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo +} + + +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} + + +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + echo "${OS_VERSION_CODENAME}" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + local PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" + curl -sSL -k -o "${PACKAGE_FILENAME}" "${PACKAGE_URL}" + sudo dpkg --install --force-depends "${PACKAGE_FILENAME}" + sudo apt install --yes --fix-broken + rm "${PACKAGE_FILENAME}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}=${VERSION}" +} + + +st2_install_dev_build() +{ + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" + + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi + + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" +} + +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="/etc/st2/st2.conf" + + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" + + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" + + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} + + +st2_configure_authentication() { + local ST2_CFGFILE="/etc/st2/st2.conf" + + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done +} + + +st2_configure_user() +{ + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi + + SYSTEM_HOME=$(echo ~stanley) + + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi + + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi + + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi + + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} + + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + + sudo chmod 0440 /etc/sudoers.d/st2 + + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers +} + + +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) + + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" + + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return + fi + + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} +} + + +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} + fi + + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done +} + + +st2_verification() +{ + echo.info "Check version" + st2 --version + + echo.info "Check help" + st2 -h + + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) + + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core + + echo.info "Check local shell command" + st2 run core.local -- date -R + + echo.info "Check execution list" + st2 execution list + + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a + + echo.info "Check pack installation" + st2 pack install st2 +} + +nodejs_configure_repository() +{ + curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - +} + +st2chatops_install() +{ + nodejs_configure_repository + pkg_install nodejs + + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} +} + +st2chatops_configure() +{ + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} + +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "nginx" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" +} + +st2web_install() +{ + nginx_configure_repo + pkg_meta_update + + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} + + sudo mkdir -p /etc/ssl/st2 + sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ + -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ + Technology/CN=$(hostname)" + + sudo rm -f /etc/nginx/conf.d/default.conf + + + + sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ + + sudo systemctl enable nginx + sudo systemctl restart nginx +} + + +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/apt/${OS_ID}" \ + "${OS_VERSION_CODENAME}/mongodb-org/7.0" \ + "multiverse" \ + "mongodb-org-7.0-key" \ + "https://www.mongodb.org/static/pgp/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongodb" + local DB_PATH="/var/lib/mongodb" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" +} + + +mongodb_write_configuration() +{ + local MONGODB_USER="$1" + local DB_PATH="$2" + local LOG_PATH="$3" + local CFGFILE="/etc/mongod.conf" + + TMP=$(cat <${CFGFILE}" +} + +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} + +mongodb_install() +{ + local MONGODB_PKG=mongodb-org + + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi + + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration + + sudo systemctl enable mongod + sudo systemctl start mongod + + sleep 10 + + mongosh </etc/apt/preferences.d/erlang.pref +Package: erlang* +Pin: origin ppa1.rabbitmq.com +Pin-Priority: 1001 +EOF + + + pkg_meta_update + pkg_install ${PKGS[@]} + + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server + + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi + + repo_definition "redis" \ + "https://packages.redis.io/deb" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "redis-key" \ + "https://packages.redis.io/gpg" + local REDIS_SERVICE=redis-server + + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi + + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" +} + +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done + +trap 'fail' EXIT + +step "Setup runtime arguments" +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password + +step "Install required runtime packages" +system_install_runtime_packages + +step "Check storage capacity and network ports" +system_check_resources + +step "Configure HTTP Proxy" +system_configure_proxy + +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords + +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" + +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" + +step "Install key/value store (Redis)" +redis_install + +step "Install st2 (StackStorm)" +st2_install + +step "Configure st2 system user account" +st2_configure_user + +step "Configure st2 authentication" +st2_configure_authentication + +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" + +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys + +step "Verify StackStorm installation" +st2_verification + +step "Install Web Interface (st2web)" +st2web_install + +step "Install ChatOps bot (st2chatops)" +st2chatops_install + +step "Configure st2chatops" +st2chatops_configure + +trap - EXIT + +ok_message diff --git a/templates/funcs_display.jinja b/templates/funcs_display.jinja new file mode 100644 index 00000000..fc4f3d36 --- /dev/null +++ b/templates/funcs_display.jinja @@ -0,0 +1,53 @@ +function centre() +{ + LINE_LEN="$1" + TEXT="$2" + OUTPUT="" + {% raw %} + if [[ ${#TEXT} -lt ${LINE_LEN} ]]; then + LS=$(( (LINE_LEN - ${#TEXT}) / 2 )) + OUTPUT+=$(printf "%0.s " $(seq 0 $LS)) + OUTPUT+="$TEXT" + RS=$(( LINE_LEN - ${#OUTPUT} )) + OUTPUT+=$(printf "%0.s " $(seq 0 $RS)) + fi + {% endraw %} + echo "${OUTPUT}" +} +# colour echo (ref https://stackoverflow.com/questions/4842424/list-of-ansi-color-escape-sequences) +function cecho() +{ + if [[ "$1" == "-n" ]]; then + # No carrage return + local NCR="$1"; shift + else + local NCR="" + fi + local C="$1"; + local MSG="$2" + echo $NCR -e "${C}${MSG}\e[0m" +} +function heading() +{ + local COLS=$(stty size | cut -d' ' -f2) + if [[ -n "$COLS" ]]; then + HEADING=$(centre $((COLS - 1)) "$1") + else + HEADING="$1" + fi + echo + cecho "\e[38;5;208m\e[48;5;238m\e[1m" "$HEADING" + echo +} +function echo.info() +{ + cecho "\e[37;1m" "$1" +} +function echo.warning() +{ + cecho "\e[33;1m" "$1" +} +function echo.error() +{ + cecho "\e[31;1m" "$1" >/dev/stderr +} diff --git a/templates/funcs_mongodb.jinja b/templates/funcs_mongodb.jinja new file mode 100644 index 00000000..911974f9 --- /dev/null +++ b/templates/funcs_mongodb.jinja @@ -0,0 +1,146 @@ +###############[ MONGODB ]############### +{% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/yum/redhat/{{ version_id }}/mongodb-org/7.0/x86_64/" \ + "mongodb-org-7.0-key" \ + "https://pgp.mongodb.com/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongod" + local DB_PATH="/var/lib/mongo" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" + mongodb_adjust_selinux_policies +} +{% elif id in ["ubuntu", "debian"] %} +{% set component = {"ubuntu": "multiverse", "debian": "main"}[id] %} +mongodb_configure_repo() +{ + repo_definition "mongodb-org-7.0" \ + "https://repo.mongodb.org/apt/${OS_ID}" \ + "${OS_VERSION_CODENAME}/mongodb-org/7.0" \ + "{{ component }}" \ + "mongodb-org-7.0-key" \ + "https://www.mongodb.org/static/pgp/server-7.0.asc" +} +mongodb_configuration() +{ + local MONGODB_USER="mongodb" + local DB_PATH="/var/lib/mongodb" + local LOG_PATH="/var/log/mongodb" + mongodb_write_configuration "$MONGODB_USER" "$DB_PATH" "$LOG_PATH" +} +{% else %} +mongodb_configure_repo() +{ + echo.error "Installing mongodb on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +} +mongodb_configuration() +{ + mongodb_configure_repo +} +{% endif %} + +mongodb_write_configuration() +{ + local MONGODB_USER="$1" + local DB_PATH="$2" + local LOG_PATH="$3" + local CFGFILE="/etc/mongod.conf" + + TMP=$(cat <${CFGFILE}" +} + +mongodb_adjust_selinux_policies() +{ + if getenforce | grep -q 'Enforcing'; then + # RHEL9 selinux policy is more restrictive than RHEL8 by default which requires + # the installation of a mongodb policy to allow it to run. + # Note that depending on distro assembly/settings you may need more rules to change + # Apply these changes OR disable selinux in /etc/selinux/config (manually) + echo.info "Applying MongoDB SELinux policy." + pkg_install git make checkpolicy policycoreutils selinux-policy-devel + test -d /root/mongodb-selinux || sudo git clone https://github.com/mongodb/mongodb-selinux /root/mongodb-selinux + cd /root/mongodb-selinux && \ + make && \ + sudo make install + fi +} + +mongodb_install() +{ + local MONGODB_PKG=mongodb-org + + if [[ $INSTALL_MONGODB -eq 0 ]]; then + echo.info "Skip MongoDB: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$MONGODB_PKG"; then + echo.info "Skip MongoDB: Package is already present on the system." + return + fi + + mongodb_configure_repo + pkg_meta_update + pkg_install "$MONGODB_PKG" + mongodb_configuration + + # Enable and restart + sudo systemctl enable mongod + sudo systemctl start mongod + + # Wait for service to come up before attempt to create user + sleep 10 + + # Create admin user and user used by StackStorm (MongoDB needs to be running) + # NOTE: mongo shell will automatically exit when piping from stdin. There is + # no need to put quit(); at the end. This way last command exit code will be + # correctly preserved and install script will correctly fail and abort if this + # command fails. + mongosh </etc/apt/preferences.d/erlang.pref +Package: erlang* +Pin: origin ppa1.rabbitmq.com +# Note: priority of 1001 (greater than 1000) allows for downgrading. +# To make package downgrading impossible, use a value of 999 +Pin-Priority: 1001 +EOF +{% else %} + echo.error "Installing erlang and rabbitmq on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +{% endif %} + + pkg_meta_update + pkg_install ${PKGS[@]} + + # Configure RabbitMQ to listen on localhost only + sudo sh -c 'echo "RABBITMQ_NODE_IP_ADDRESS=127.0.0.1" >> /etc/rabbitmq/rabbitmq-env.conf' + + sudo systemctl enable rabbitmq-server + sudo systemctl restart rabbitmq-server + + # configure RabbitMQ + if ! sudo rabbitmqctl list_users | grep -E '^stackstorm'; then + sudo rabbitmqctl add_user stackstorm "${ST2_RABBITMQ_PASSWORD}" + sudo rabbitmqctl set_user_tags stackstorm administrator + sudo rabbitmqctl set_permissions -p / stackstorm ".*" ".*" ".*" + fi + if sudo rabbitmqctl list_users | grep -E '^guest'; then + sudo rabbitmqctl delete_user guest + fi +} diff --git a/templates/funcs_redis.jinja b/templates/funcs_redis.jinja new file mode 100644 index 00000000..821f6ed5 --- /dev/null +++ b/templates/funcs_redis.jinja @@ -0,0 +1,59 @@ +###############[ REDIS ]############### +redis_install() +{ + local REDIS_PKG=redis + + if [[ $INSTALL_REDIS -eq 0 ]]; then + echo.info "Skip Redis: Installation explicitly disabled at runtime." + return + elif pkg_is_installed "$REDIS_PKG"; then + echo.info "Skip Redis: Package is already present on the system." + return + fi +{% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} + # https://redis.io/docs/latest/operate/oss_and_stack/install/archive/install-redis/install-redis-on-linux/#install-on-red-hatrocky + # use system provided packages. + local REDIS_SERVICE=redis +{% elif id in ["ubuntu", "debian"] %} + # https://redis.io/docs/latest/operate/oss_and_stack/install/archive/install-redis/install-redis-on-linux/ + repo_definition "redis" \ + "https://packages.redis.io/deb" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "redis-key" \ + "https://packages.redis.io/gpg" + local REDIS_SERVICE=redis-server +{% else %} + echo.error "Installing redis-server on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +{% endif %} + pkg_meta_update + pkg_install "$REDIS_PKG" + + TMP=$(cat </etc/redis.conf" + elif [[ -f /etc/redis/redis.conf ]]; then + # redis v6 configuration + sudo bash -c "cat <<<\"$TMP\" >/etc/redis/redis.conf" + else + echo.warning "Unable to find redis configuration file at /etc/redis.conf or /etc/redis/redis.conf." + fi + + sudo systemctl enable "${REDIS_SERVICE}" + sudo systemctl start "${REDIS_SERVICE}" +} diff --git a/templates/funcs_repo_manager.jinja b/templates/funcs_repo_manager.jinja new file mode 100644 index 00000000..71a5611c --- /dev/null +++ b/templates/funcs_repo_manager.jinja @@ -0,0 +1,143 @@ +###############[ REPOSITORY MANAGER FUNCTIONS ]############### +{# filter functions based on package manager. #} +{% if pkg_mgr == "dnf" %} +pkg_get_latest_version() +{ + local PKG="$1" # st2 + local VERSION="$2" # 3.9dev + LATEST=$(repoquery -y --nvr --show-duplicates "$PKG" | grep -F "${PKG}-${VERSION}" | sort --version-sort | tail -n 1) + echo "${LATEST#*-}" +} + + +repo_add_gpg_key() +{ + KEY_NAME="$1" + KEY_URL="$2" + rpm --import "${KEY_URL}" +} + + +repo_definition() +{ + REPO_NAME="$1" + REPO_URL="$2" + KEY_NAME="$3" + KEY_URL="$4" + REPO_PATH="/etc/yum.repos.d/" + + cat <"${REPO_PATH}/${REPO_NAME}.repo" +[${REPO_NAME}] +name=${REPO_NAME} +baseurl=${REPO_URL} +repo_gpgcheck=1 +enabled=1 +gpgkey=${KEY_URL} +gpgcheck=0 +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +pkg_gpgcheck=1 +autorefresh=1 +type=rpm-md +EOF +} + + +pkg_get_versions() +{ + # Approximately equivalanet to apt-cache show + dnf -y info --showduplicates "$1" + # output processing hint (not completely correct): + # dnf info --showduplicates st2 | sed -r 's/ +: +/:/g' | awk -F: '/^(Name|Version|Release|Architecture|Size|Source|Repository|Summary|URL|License|Description)/ {print $2} | xargs -n11 | column -t +} + + +repo_clean_meta() +{ + dnf -y clean metadata + dnf -y clean dbcache + dnf -y clean all +} +{% elif pkg_mgr == "apt" %} +{% set gpg_key_path = "/etc/apt/trusted.gpg.d" %} +repo_definition() +{ + REPO_PATH="/etc/apt/sources.list.d" + GPG_KEY_PATH="{{ gpg_key_path }}" + + REPO_NAME="$1" + REPO_URL="$2" + REPO_SUITES="$3" + REPO_COMPONENT="$4" + KEY_NAME="$5" + KEY_URL="$6" + + repo_add_gpg_key "$KEY_NAME" "$KEY_URL" + # DEB822 is preferred over list format. + sudo cat <"${REPO_PATH}/${REPO_NAME}.sources" +Types: deb +URIs: ${REPO_URL} +Suites: ${REPO_SUITES} +Components: ${REPO_COMPONENT} +Architectures: $(dpkg --print-architecture) +Signed-By: ${GPG_KEY_PATH}/${KEY_NAME}.gpg +EOF +} + + +repo_add_gpg_key() +{ + GPG_KEY_PATH="{{ gpg_key_path }}" + KEY_NAME="$1" + KEY_URL="$2" + + curl -1sLf "$KEY_URL" | sudo gpg --dearmor -o "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" + sudo mv "${GPG_KEY_PATH}/${KEY_NAME}.gpg.tmp" "${GPG_KEY_PATH}/${KEY_NAME}.gpg" +} + + +pkg_get_latest_version() +{ + local PKG="$1" + local VERSION="$2" + apt-cache show "$PKG" | awk '/Version:/{print $2}' | grep "^${VERSION//./\\.}" | sort --version-sort | tail -n 1 +} + + +pkg_get_versions() +{ + apt-cache show "$1" +} + + +repo_clean_meta() +{ + true +} +{% endif %} + +repo_pkg_availability() { + # repo_pkg_availability + local PKG="$1" + local VERSION="$2" + + local PKG_VER="" + {% if pkg_mgr == "dnf" %} + # rpm based systems. + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + {% elif pkg_mgr == "apt" %} + # deb based systems. + PKG_VER=$(pkg_get_latest_version "$PKG" "${VERSION}") + {% else %} + # unsupported/unknown package managers. + echo.error "Failed to detect a supported repository query tool (apt-cache or repoquery)." + exit 1 + {% endif %} + + if [[ -z "$PKG_VER" ]]; then + echo.error "${PKG}-${VERSION} couldn't be found in the pacakges available on this system." + exit 3 + fi + echo "$PKG_VER" +} diff --git a/templates/funcs_setup.jinja b/templates/funcs_setup.jinja new file mode 100644 index 00000000..200f2f70 --- /dev/null +++ b/templates/funcs_setup.jinja @@ -0,0 +1,92 @@ +###############[ SCRIPT PARAMETER SETUP ]############### +setup_install_parameters() +{ + # Valid release repository combinations: + # stable with version x.y.z + # https://packagecloud.io/StackStorm/stable (st2web-3.8.1-1.x86_64.rpm) + # staging-stable with version x.y.z + # https://packagecloud.io/StackStorm/staging-stable (st2chatops_3.8.1-1_amd64.deb) + # unstable with version x.ydev + # https://packagecloud.io/StackStorm/unstable (st2-3.9dev-208.x86_64.rpm) + # staging-unstable with version x.ydev + # https://packagecloud.io/StackStorm/staging-unstable (st2-3.9dev-97.x86_64.rpm) + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + local DEV_BUILD="$4" + + # Set the installation type to use in the script. + if [[ -n "$DEV_BUILD" ]]; then + # Development builds use the package produced from CI directly. + # https://output.circle-artifacts.com/output/job/e404c552-f8d6-46bd-9034-0267148874db/artifacts/0/packages/focal/st2_3.9dev-186_amd64.deb + # CircleCI pipeline repo: st2, branch: master, workflow: package-test-and-deploy, job: 17505 + INSTALL_TYPE["CI"]="$DEV_BUILD" + if [[ ! "$DEV_BUILD" =~ [^/]+/[0-9]+ ]]; then + echo.error "Unexpected format '$DEV_BUILD'. Format must be 'repo_name/build_id'" + exit 1 + fi + echo.warning "Installation of $DEV_BUILD from CI build artifacts! REALLY, ANYTHING COULD HAPPEN!" + else + # non-development builds use the PackageCloud repositories. + setup_select_repository "$VERSION" "$RELEASE" "$REPO_TYPE" + fi +} + + +setup_check_version() +{ + # StackStorm version sanity check. Report and error and exit if + # the version doesn't conform to the format x.y.z or x.ydev + local VERSION="$1" + if [[ -z "$VERSION" ]]; then + echo.error "Unable to run script because no StackStorm version was provided." + usage + exit 1 + fi + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + echo.error "$VERSION does not match supported formats x.y.z or x.ydev." + exit 1 + fi +} + + +setup_select_repository() +{ + local VERSION="$1" + local RELEASE="$2" + local REPO_TYPE="$3" + + setup_check_version "$VERSION" + + # Version takes precedence over requested release + if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+dev$ ]]; then + if [[ "$RELEASE" != "unstable" ]]; then + echo.warning "Development version $VERSION requested, switching from '$RELEASE' to 'unstable' repository!" + RELEASE='unstable' + fi + fi + + if [[ -n "$REPO_TYPE" ]]; then + echo.warning "Installing from staging repository: USE AT YOUR OWN RISK!" + RELEASE="${REPO_TYPE}-${RELEASE}" + fi + echo.info "Installation of StackStorm $VERSION from repository $RELEASE." + INSTALL_TYPE["REPO"]="$RELEASE" +} + + +setup_username_password() +{ + if [[ "$USERNAME" = '' || "$PASSWORD" = '' ]]; then + echo "Let's set StackStorm admin credentials." + echo 'You can also use "--user" and "--password" for unattended installation.' + echo 'Press to continue or to exit/abort.' + read -e -p "Admin username: " -i "st2admin" USERNAME + read -e -s -p "Password: " PASSWORD + echo + if [[ -z "${PASSWORD}" ]]; then + echo.error "Password cannot be empty." + exit 1 + fi + fi +} diff --git a/templates/funcs_st2.jinja b/templates/funcs_st2.jinja new file mode 100644 index 00000000..d3e67873 --- /dev/null +++ b/templates/funcs_st2.jinja @@ -0,0 +1,327 @@ +###############[ STACKSTORM ]############### +{% set st2conf = "/etc/st2/st2.conf" %} +{% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/el/{{ version_id }}/\$basearch/" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + # Use jinja version id for major version only. + echo "el{{ version_id }}" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + pkg_install "${PACKAGE_URL}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}-${VERSION}" +} +{% elif id in ["ubuntu", "debian"] %} +st2_configure_repository() +{ + local REPO_TGT="$1" + repo_definition "st2-${REPO_TGT}" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "main" \ + "st2-${REPO_TGT}-key" \ + "https://packagecloud.io/StackStorm/${REPO_TGT}/gpgkey" +} +st2_distribution_name() +{ + # codename is used rather than version id. + echo "${OS_VERSION_CODENAME}" +} +st2_install_from_url() +{ + local PACKAGE_URL="$1" + local PACKAGE_FILENAME="$(basename ${PACKAGE_URL})" + curl -sSL -k -o "${PACKAGE_FILENAME}" "${PACKAGE_URL}" + sudo dpkg --install --force-depends "${PACKAGE_FILENAME}" + sudo apt install --yes --fix-broken + rm "${PACKAGE_FILENAME}" +} +st2_install_pkg_version() +{ + local PKG="$1" + local VERSION="$2" + pkg_install "${PKG}=${VERSION}" +} +{% else %} +st2_configure_repository() +{ + echo.error "Installing st2 on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +} +st2_distribution_name() +{ + st2_configure_repository +} +st2_install_from_local() +{ + st2_configure_repository +} +st2_install_pkg_version() +{ + st2_configure_repository +} +{% endif %} + +st2_install_dev_build() +{ + # Retrieve package URL for the provided dev build from CircleCI build pipeline. + DEV_BUILD="$1" # Repo name and build number - / (e.g. st2/5646) + DISTRO="$(st2_distribution_name)" # Distro name (e.g. focal, jammy, el8, el9) + PACKAGE_NAME_REGEX="${DISTRO}/st2[_-].*\.(deb|rpm)$" + MANIFEST_URL="https://circleci.com/api/v1.1/project/github/StackStorm/${DEV_BUILD}/artifacts" + + PACKAGES_METADATA=$(curl -sSL -q "$MANIFEST_URL" || true) + if [[ -z "${PACKAGES_METADATA}" ]]; then + echo.error "Failed to retrieve packages metadata from $MANIFEST_URL" + exit 30 + fi + + ARTIFACT_URLS=$(jq -r '.[].url' <<<"$PACKAGES_METADATA" || true) + if [[ -z "$ARTIFACT_URLS" ]]; then + echo.error "No urls found in manifest. This might be because the JSON structure changed or is invalid." + exit 31 + fi + + PACKAGE_URL=$(grep -E "${PACKAGE_NAME_REGEX}" <<<"$ARTIFACT_URLS" || true) + if [[ -z "${PACKAGE_URL}" ]]; then + echo.error "Failed to find url for ${DISTRO} package (${PACKAGE_NAME_REGEX})" + echo.error "Circle CI response: ${PACKAGES_METADATA}" + exit 32 + fi + echo.info "Installing CI artifact from ${PACKAGE_URL}" + st2_install_from_url "$PACKAGE_URL" +} + +st2_install() +{ + if [[ "${!INSTALL_TYPE[@]}" == "REPO" ]]; then + st2_configure_repository "${INSTALL_TYPE[REPO]}" + pkg_meta_update + + ST2_PKG_VERSION="$(repo_pkg_availability st2 $VERSION)" + ST2WEB_PKG_VERSION="$(repo_pkg_availability st2web $VERSION)" + ST2CHATOPS_PKG_VERSION="$(repo_pkg_availability st2chatops $VERSION)" + + echo.info "The following versions of packages will be installed" + echo.info " ${ST2_PKG_VERSION}" + echo.info " ${ST2WEB_PKG_VERSION}" + echo.info " ${ST2CHATOPS_PKG_VERSION}" + st2_install_pkg_version st2 ${ST2_PKG_VERSION} + + elif [[ "${!INSTALL_TYPE[@]}" == "CI" ]]; then + echo.info "Development build ${INSTALL_TYPE[CI]}" + st2_install_dev_build "${INSTALL_TYPE[CI]}" + else + echo.error "Unknown installation type ${!INSTALL_TYPE[@]}." + exit 3 + fi + + local ST2_CFGFILE="{{ st2conf }}" + + # Configure [database] section in st2.conf (username password for MongoDB access) + local DB_URI="mongodb://stackstorm:${ST2_MONGODB_PASSWORD}@127.0.0.1:27017/st2?authSource=st2" + sudo crudini --set "$ST2_CFGFILE" database host "$DB_URI" + + # Configure [messaging] section in st2.conf (username password for RabbitMQ access) + local AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" + sudo crudini --set "$ST2_CFGFILE" messaging url "${AMQP}" + + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set "$ST2_CFGFILE" coordination url "redis://127.0.0.1:6379" + + if [[ ! -d /var/log/st2 ]]; then + echo.warning "Work around packging bug: create /var/log/st2" + sudo mkdir -p /var/log/st2 + sudo chown st2 /var/log/st2 + fi + sudo st2ctl reload --register-all + sudo st2ctl restart +} + + +st2_configure_authentication() { + local ST2_CFGFILE="{{ st2conf }}" + + # Create a user record in a password file. + sudo htpasswd -i /etc/st2/htpasswd $USERNAME <<<"${PASSWORD}" + + # Configure [auth] section in st2.conf + sudo crudini --set "$ST2_CFGFILE" auth enable "True" + sudo crudini --set "$ST2_CFGFILE" auth backend "flat_file" + sudo crudini --set "$ST2_CFGFILE" auth backend_kwargs '{"file_path": "/etc/st2/htpasswd"}' + + for srv in st2auth st2api st2stream + do + sudo st2ctl restart-component $srv + done +} + + +st2_configure_user() +{ + # Create an SSH system user (default `stanley` user may be already created) + if (! id stanley 2>/dev/null); then + sudo useradd stanley + fi + + SYSTEM_HOME=$(echo ~stanley) + + if [ ! -d "${SYSTEM_HOME}/.ssh" ]; then + sudo mkdir ${SYSTEM_HOME}/.ssh + sudo chmod 700 ${SYSTEM_HOME}/.ssh + fi + + # Generate ssh keys on StackStorm box and copy over public key into remote box. + # NOTE: If the file already exists and is non-empty, then assume the key does not need + # to be generated again. + if ! sudo test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + # added PEM to enforce PEM ssh key type in EL8 to maintain consistency + sudo ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" -m PEM + fi + + if ! sudo grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; + then + # Authorize key-base access + sudo sh -c "cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys" + fi + + sudo chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + sudo chmod 0700 ${SYSTEM_HOME}/.ssh + sudo chown -R stanley:stanley ${SYSTEM_HOME} + + # Enable passwordless sudo + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! sudo grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2" + fi + + sudo chmod 0440 /etc/sudoers.d/st2 + + # Disable requiretty for all users + sudo sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers +} + + +st2_configure_cli_config() +{ + local USERNAME="$1" + local PASSWORD="$2" + test -z "$USERNAME" && ( echo.error "Can't configure cli, missing username."; exit 9 ) + test -z "$PASSWORD" && ( echo.error "Can't configure cli, missing password."; exit 9 ) + + # Configure CLI config (write credentials for the root user and user which ran the script) + ROOT_USER="root" + CURRENT_USER=$(whoami) + + ROOT_HOME=$(eval echo ~${ROOT_USER}) + : "${HOME:=$(eval echo ~${CURRENT_USER})}" + + ROOT_USER_CLI_CONFIG_DIRECTORY="${ROOT_HOME}/.st2" + ROOT_USER_CLI_CONFIG_PATH="${ROOT_USER_CLI_CONFIG_DIRECTORY}/config" + + CURRENT_USER_CLI_CONFIG_DIRECTORY="${HOME}/.st2" + CURRENT_USER_CLI_CONFIG_PATH="${CURRENT_USER_CLI_CONFIG_DIRECTORY}/config" + + if ! sudo test -d ${ROOT_USER_CLI_CONFIG_DIRECTORY}; then + sudo mkdir -p ${ROOT_USER_CLI_CONFIG_DIRECTORY} + fi + + # Write config for root user + sudo sh -c "cat <${ROOT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + # Write config for current user (in case current user is not the root user) + if [ "${CURRENT_USER}" == "${ROOT_USER}" ]; then + return + fi + + if [ ! -d ${CURRENT_USER_CLI_CONFIG_DIRECTORY} ]; then + sudo mkdir -p ${CURRENT_USER_CLI_CONFIG_DIRECTORY} + fi + + sudo sh -c "cat < ${CURRENT_USER_CLI_CONFIG_PATH} +[credentials] +username = ${USERNAME} +password = ${PASSWORD} +EOF" + + # Fix the permissions + sudo chown -R ${CURRENT_USER}:${CURRENT_USER} ${CURRENT_USER_CLI_CONFIG_DIRECTORY} +} + + +st2_setup_kvstore_encryption_keys() +{ + DATASTORE_ENCRYPTION_KEYS_DIRECTORY="/etc/st2/keys" + DATASTORE_ENCRYPTION_KEY_PATH="${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}/datastore_key.json" + + sudo mkdir -p ${DATASTORE_ENCRYPTION_KEYS_DIRECTORY} + + # If the file ${DATASTORE_ENCRYPTION_KEY_PATH} exists and is not empty, then do not generate + # a new key. st2-generate-symmetric-crypto-key fails if the key file already exists. + if ! sudo test -s ${DATASTORE_ENCRYPTION_KEY_PATH}; then + sudo st2-generate-symmetric-crypto-key --key-path ${DATASTORE_ENCRYPTION_KEY_PATH} + fi + + # Make sure only st2 user can read the file + for dir in "${DATASTORE_ENCRYPTION_KEYS_DIRECTORY}" "${DATASTORE_ENCRYPTION_KEY_PATH}" + do + sudo chgrp st2 "$dir" + sudo chmod o-r "${dir}" + done + # set path to the key file in the config + sudo crudini --set /etc/st2/st2.conf keyvalue encryption_key_path ${DATASTORE_ENCRYPTION_KEY_PATH} + + # NOTE: We need to restart all the affected services so they pick the key and load it in memory + for srv in st2api st2sensorcontainer st2workflowengine st2actionrunner + do + sudo st2ctl restart-component $srv + done +} + + +st2_verification() +{ + echo.info "Check version" + st2 --version + + echo.info "Check help" + st2 -h + + echo.info "Check Authentication" + st2 auth $USERNAME -p $PASSWORD + # A shortcut to authenticate and export the token + export ST2_AUTH_TOKEN=$(st2 auth $USERNAME -p $PASSWORD -t) + + echo.info "Check actions list for 'core' pack" + st2 action list --pack=core + + echo.info "Check local shell command" + st2 run core.local -- date -R + + echo.info "Check execution list" + st2 execution list + + echo.info "Check remote comand via SSH (Requires passwordless SSH)" + st2 run core.remote hosts='127.0.0.1' -- uname -a + + echo.info "Check pack installation" + st2 pack install st2 +} diff --git a/templates/funcs_st2chatops.jinja b/templates/funcs_st2chatops.jinja new file mode 100644 index 00000000..74f0b6d8 --- /dev/null +++ b/templates/funcs_st2chatops.jinja @@ -0,0 +1,62 @@ +###############[ ST2CHATOPS ]############### +{% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} +nodejs_configure_repository() +{ + curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo -E bash - +} +{% elif id in ["ubuntu", "debian"] %} +nodejs_configure_repository() +{ + curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - +} +{% else %} +nodejs_configure_repository() +{ + echo.error "Installing NodeJS on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +} +{% endif %} +st2chatops_install() +{ + # Add NodeJS 20 repo + nodejs_configure_repository + pkg_install nodejs + + # Install st2chatops + st2_install_pkg_version st2chatops ${ST2CHATOPS_PKG_VERSION} +} + +st2chatops_configure() +{ + # set API keys. This should work since CLI is configuered already. + ST2_API_KEY=$(st2 apikey create -k) + sudo sed -i -r "s/^(export ST2_API_KEY.).*/\1$ST2_API_KEY/" /opt/stackstorm/chatops/st2chatops.env + + sudo sed -i -r "s/^(export ST2_AUTH_URL.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_USERNAME.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export ST2_AUTH_PASSWORD.).*/# &/" /opt/stackstorm/chatops/st2chatops.env + + # Setup adapter + if [[ "$HUBOT_ADAPTER"="slack" ]] && [[ ! -z "$HUBOT_SLACK_BOT_TOKEN" ]] && [[ ! -z "$HUBOT_SLACK_APP_TOKEN" ]]; + then + sudo sed -i -r "s/^# (export HUBOT_ADAPTER=slack)/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_BOT_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^# (export HUBOT_SLACK_APP_TOKEN.).*/\1/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_ADAPTER.).*/\1$HUBOT_ADAPTER/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_BOT_TOKEN.).*/\1$HUBOT_SLACK_BOT_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + sudo sed -i -r "s/^(export HUBOT_SLACK_APP_TOKEN.).*/\1$HUBOT_SLACK_APP_TOKEN/" /opt/stackstorm/chatops/st2chatops.env + + sudo service st2chatops restart + else + echo.warning "Warning: Chatops requires manual configuration!" + echo.info "Edit /opt/stackstorm/chatops/st2chatops.env to specify" + echo.info "the adapter and settings hubot should use to connect to" + echo.info "the chat you're using. Don't forget to start the service" + echo.info "afterwards:" + echo.info "" + echo.info " $ sudo systemctl restart st2chatops" + echo.info "" + echo.info "For more information, please refer to documentation at" + echo.info "https://docs.stackstorm.com/install/index.html" + fi +} diff --git a/templates/funcs_st2web.jinja b/templates/funcs_st2web.jinja new file mode 100644 index 00000000..8a4756e8 --- /dev/null +++ b/templates/funcs_st2web.jinja @@ -0,0 +1,63 @@ +###############[ ST2WEB ]############### +{% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/rhel/{{ version_id }}/x86_64/" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" + + # Ensure that EPEL repo is not used for nginx (to do: confirm this is still needed) + #~ sudo sed -i 's/^\(enabled=1\)$/exclude=nginx\n\1/g' /etc/yum.repos.d/epel.repo +} +{% elif id in ["ubuntu", "debian"] %} +nginx_configure_repo() +{ + repo_definition "nginx" \ + "http://nginx.org/packages/${OS_ID}" \ + "${OS_VERSION_CODENAME}" \ + "nginx" \ + "nginx-key" \ + "http://nginx.org/keys/nginx_signing.key" +} +{% else %} +nginx_configure_repo() +{ + echo.error "Installing nginx on $OS_ID $OS_VERSION isn't supported by this script." + exit 4 +} +{% endif %} +st2web_install() +{ + nginx_configure_repo + pkg_meta_update + + pkg_install nginx + st2_install_pkg_version st2web ${ST2WEB_PKG_VERSION} + + # Generate self-signed certificate or place your existing certificate under /etc/ssl/st2 + sudo mkdir -p /etc/ssl/st2 + sudo openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt \ + -days 365 -nodes -subj "/C=US/ST=California/L=Palo Alto/O=StackStorm/OU=Information \ + Technology/CN=$(hostname)" + + # Remove default site, if present + sudo rm -f /etc/nginx/conf.d/default.conf + + {% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} + # back up conf + sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak + # comment out server block eg. server {...} + sudo awk '/^ server {/{f=1}f{$0 = "#" $0}{print}' /etc/nginx/nginx.conf.bak >/etc/nginx/nginx.conf + # remove double comments + sudo sed -i -e 's/##/#/' /etc/nginx/nginx.conf + # remove comment closing out server block + sudo sed -i -e 's/#}/}/' /etc/nginx/nginx.conf + {% endif %} + + # Copy and enable StackStorm's supplied config file + sudo cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/ + + sudo systemctl enable nginx + sudo systemctl restart nginx +} diff --git a/templates/funcs_system.jinja b/templates/funcs_system.jinja new file mode 100644 index 00000000..3f484389 --- /dev/null +++ b/templates/funcs_system.jinja @@ -0,0 +1,243 @@ +###############[ COMMON FUNCTIONS ]############### +system_install_runtime_packages() +{ + {% if id in ["rhel", "rocky", "centos"] %} + # Extra Packages for Enterprise Linux (EPEL) for crudini requirement + if ! pkg_is_installed epel-release; then + pkg_install https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ version_id }}.noarch.rpm + fi + {% endif %} + local PKG_DEPS=( + crudini + curl + jq + logrotate + net-tools + {% if id in ["rhel", "rocky", "centos", "opensuse-leap"] -%} + # Use repoquery tool from yum-utils to get package_name-package_ver-package_rev in RPM based distros + # if we don't want to construct this string manually using yum info --show-duplicates and + # doing a bunch of sed awk magic. Problem is this is not installed by default on all images. + yum-utils + iproute + gnupg2 + httpd-tools + {% elif id in ["ubuntu", "debian"] %} + iproute2 + gnupg + apt-transport-https + apache2-utils + ca-certificates + {% if id == "debian" %} + debian-archive-keyring + {% endif %} + {% endif -%} + ) + pkg_meta_update + pkg_install ${PKG_DEPS[@]} +} + + +system_configure_proxy() +{ + # Allow bypassing 'proxy' env vars via sudo + local sudoers_proxy='Defaults env_keep += "http_proxy https_proxy no_proxy proxy_ca_bundle_path DEBIAN_FRONTEND"' + if ! sudo grep -s -q ^"${sudoers_proxy}" /etc/sudoers.d/st2; then + sudo sh -c "echo '${sudoers_proxy}' >> /etc/sudoers.d/st2" + fi + + # Configure proxy env vars for 'st2api', 'st2actionrunner' and 'st2chatops' system configs + # See: https://docs.stackstorm.com/packs.html#installing-packs-from-behind-a-proxy + service_config_path="" + # sysconfig and default exist on RedHat systems, so sysconfig must be first in the search list. + for cfgdir in "/etc/sysconfig" "/etc/default" + do + if [[ -d "$cfgdir" ]]; then + service_config_path="$cfgdir" + break + fi + done + + if [[ -z "$service_config_path" ]]; then + echo.error "Failed to determine the systems configuration path! Is this system supported?" + exit 1 + fi + for service in st2api st2actionrunner st2chatops; + do + service_config="${service_config_path}/${service}" + # create file if doesn't exist yet + sudo test -e "${service_config}" || sudo touch "${service_config}" + for env_var in http_proxy https_proxy no_proxy proxy_ca_bundle_path; do + # delete line from file if specific proxy env var is unset + if sudo test -z "${!env_var:-}"; then + sudo sed -i "/^${env_var}=/d" ${service_config} + # add proxy env var if it doesn't exist yet + elif ! sudo grep -s -q ^"${env_var}=" ${service_config}; then + sudo sh -c "echo '${env_var}=${!env_var}' >> ${service_config}" + # modify existing proxy env var value + elif ! sudo grep -s -q ^"${env_var}=${!env_var}$" ${service_config}; then + sudo sed -i "s#^${env_var}=.*#${env_var}=${!env_var}#" ${service_config} + fi + done + done +} + + +system_port_status() +{ + # If the specified tcp4 port is bound, then return the "port pid/procname", + # else if a pipe command fails, return "Unbound", + # else return "". + # + # Please note that all return values end with a newline. + # + # Use netstat and awk to get a list of all the tcp4 sockets that are in the LISTEN state, + # matching the specified port. + # + # `ss` command is expected to output data in the below format: + # Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port Process + # tcp LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("nginx",pid=421,fd=6),("nginx",pid=420,fd=6)) + + # The awk command prints the 4th and 7th columns of any line matching both the following criteria: + # 1) The 5th column contains the port passed to port_status() (i.e., $1) + # 2) The 7th column contains the process bound (listening) to the port. + # + # Sample output: + # 0.0.0.0:80 users:(("nginx",pid=421,fd=6),("nginx",pid=420,fd=6)) + sudo ss -ltpun4 "sport = :$1" | awk '/tcp.*LISTEN.*/ {print $5" "$7}' || echo "Unbound" +} + + +system_check_resources() +{ + # CHECK 1: Determine which, if any, of the required ports are used by an existing process. + + # Abort the installation early if the following ports are being used by an existing process. + # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) + # and st2 (9100-9102). + PORT_TEST=$( + cat </root/st2_credentials +User account details: + StackStorm + username: $USERNAME + password: $PASSWORD + MongoDB + username: admin + password: $ST2_MONGODB_PASSWORD + RabbitMQ + username: stackstorm + password: $ST2_RABBITMQ_PASSWORD +EOF +} + +step() +{ + export STEP="$1" + echo; heading "$STEP"; echo +} + + +fail() +{ + echo.error "Failed during '$STEP'" + exit 2 +} diff --git a/templates/funcs_usage.jinja b/templates/funcs_usage.jinja new file mode 100644 index 00000000..4917c61f --- /dev/null +++ b/templates/funcs_usage.jinja @@ -0,0 +1,47 @@ +###############[ SCRIPT HELP ]############### +usage() { + cat <] [--stable|--unstable] [--staging] [--dev=] [--user=] [--password=] + [--no-mongodb] [--no-rabbitmq] [--no-redis] [--no-st2chatops] [--no-st2web] + + StackStorm installation script. This script will configure and install StackStorm and its dependencies on the system. + WARNING: This script will make system changes that aren't automatically reversible. + + Parameters + --version|-v: The StackStorm version to be installed. + Stable versions are ... E.g. --version=3.8.1 to install StackStorm v3.8.1 from the stable repository. + Unstable versions are .dev. E.g. --version=3.9dev to install the latest StackStorm v3.9dev from the unstable repository. + + --username: The StackStorm account name to be created. + + --password: The password for the StackStorm account. + + --stable|-s: Install StackStorm packages from the stable repository. (default) + Packages are officially supported and production ready. + The stable option is mutually exclusive with the unstable option. + + --unstable|-u: Install StackStorm packages from the unstable repository. + Daily or Promoted packages built after passing end-to-end testing from the StackStorm development branch. + + --staging: Install StackStorm packages from the staging- repository. + This option is combined with the stable/unstable option. + staging-stable packages are release candidate made available for testing during the StackStorm release process. + staging-unstable experimental packages that are built from the latest development branch that have passed unit testing. + + --dev=*: Install StackStorm from Continuous Integration artifact. + The pamameter takes the git repository name and build number - /. E.g. --dev=st2/5646 + Do not use this option unless you understand what you're doing. + + --no-mongodb Disable the installation procedure for MongoDB on the system. + + --no-rabbitmq Disable the installation procedure for RabbitMQ on the system. + + --no-redis Disable the installation procedure for Redis on the system. + + --no-st2chatops Disable the installation procedure for st2 chatops on the system. + + --no-st2web Disable the installation procedure for st2 web ui on the system. + +EOF +} diff --git a/templates/st2bootstrap.jinja b/templates/st2bootstrap.jinja new file mode 100644 index 00000000..eaad3a8f --- /dev/null +++ b/templates/st2bootstrap.jinja @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# +# DO NOT EDIT MANUALLY. GENERATED FOR {{ id }} {{ version_id }} +# +# Please edit the corresponding template file and include files in https://github.com/StackStorm/st2-packages.git. + +set -e -u +x + +# ============================ Global variables ============================ +HUBOT_ADAPTER='slack' +HUBOT_SLACK_BOT_TOKEN=${HUBOT_SLACK_BOT_TOKEN:-''} +HUBOT_SLACK_APP_TOKEN=${HUBOT_SLACK_APP_TOKEN:-''} +VERSION='' +RELEASE='stable' +REPO_TYPE='' +DEV_BUILD='' +USERNAME='' +PASSWORD='' +ST2_PKG='st2' +ST2WEB_PKG='st2web' +ST2CHATOPS_PKG='st2chatops' +INSTALL_MONGODB=1 +INSTALL_RABBITMQ=1 +INSTALL_REDIS=1 +INSTALL_ST2CHATOPS=1 +INSTALL_ST2WEB=1 + +declare -A INSTALL_TYPE=() +#~ export INSTALL_TYPE + +# Prefix operating system variables with OS_ to avoid conflicts in the script. +# References: https://github.com/chef/os_release +source <(sed 's/^/OS_/g' /etc/os-release) + +# ============================ Function declarations ============================ +{% include 'funcs_usage.jinja' %} +{% include 'funcs_display.jinja' %} +{% include 'funcs_setup.jinja' %} +{% include 'funcs_package_manager.jinja' %} +{% include 'funcs_repo_manager.jinja' %} +{% include 'funcs_system.jinja' %} +{% include 'funcs_st2.jinja' %} +{% include 'funcs_st2chatops.jinja' %} +{% include 'funcs_st2web.jinja' %} +{% include 'funcs_mongodb.jinja' %} +{% include 'funcs_rabbitmq.jinja' %} +{% include 'funcs_redis.jinja' %} + +# ============================ Main script logic ============================ +for i in "$@" +do + case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -s|--stable) + RELEASE=stable + shift + ;; + -u|--unstable) + RELEASE=unstable + shift + ;; + --staging) + REPO_TYPE='staging' + shift + ;; + --dev=*) + DEV_BUILD="${i#*=}" + shift + ;; + --user=*|--username=*) + USERNAME="${i#*=}" + shift + ;; + --password=*) + PASSWORD="${i#*=}" + shift + ;; + --no-mongodb) + INSTALL_MONGODB=0 + shift + ;; + --no-rabbitmq) + INSTALL_RABBITMQ=0 + shift + ;; + --no-redis) + INSTALL_REDIS=0 + shift + ;; + --no-st2chatops) + INSTALL_ST2CHATOPS=0 + shift + ;; + --no-st2web) + INSTALL_ST2WEB=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown parameter $i." + usage + exit 1 + ;; + esac +done + +trap 'fail' EXIT + +step "Setup runtime arguments" +# Side-effect: INSTALL_TYPE is updated from setup_install_parameters() +setup_install_parameters "$VERSION" "$RELEASE" "$REPO_TYPE" "$DEV_BUILD" +setup_username_password + +step "Install required runtime packages" +system_install_runtime_packages + +step "Check storage capacity and network ports" +system_check_resources + +step "Configure HTTP Proxy" +system_configure_proxy + +ST2_RABBITMQ_PASSWORD=$(system_generate_password 24) +ST2_MONGODB_PASSWORD=$(system_generate_password 24) +write_passwords + +step "Install event bus (RabbitMQ)" +rabbitmq_install "$ST2_RABBITMQ_PASSWORD" + +step "Install database (MongoDB)" +mongodb_install "$ST2_MONGODB_PASSWORD" + +step "Install key/value store (Redis)" +redis_install + +step "Install st2 (StackStorm)" +st2_install + +step "Configure st2 system user account" +st2_configure_user + +step "Configure st2 authentication" +st2_configure_authentication + +step "Create st2 CLI configuration" +st2_configure_cli_config "$USERNAME" "$PASSWORD" + +step "Setup datastore symmetric encryption" +st2_setup_kvstore_encryption_keys + +step "Verify StackStorm installation" +st2_verification + +step "Install Web Interface (st2web)" +st2web_install + +step "Install ChatOps bot (st2chatops)" +st2chatops_install + +step "Configure st2chatops" +st2chatops_configure + +trap - EXIT + +ok_message diff --git a/tools/generate_final_installer_scripts.py b/tools/generate_final_installer_scripts.py deleted file mode 100755 index 0dfcbae2..00000000 --- a/tools/generate_final_installer_scripts.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 - -import os - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_PATH = os.path.abspath(os.path.join(BASE_DIR, '../scripts')) - -COMMON_INCLUDE_PATH = os.path.join(SCRIPTS_PATH, 'includes/common.sh') -RHEL_INCLUDE_PATH = os.path.join(SCRIPTS_PATH, 'includes/rhel.sh') - -SCRIPT_FILES = [ - 'st2bootstrap-deb.sh', - 'st2bootstrap-el8.sh', - 'st2bootstrap-el9.sh' -] - -HEADER_WARNING = """ -#!/usr/bin/env bash -# NOTE: This file is automatically generated by the tools/generate_final_installer_scripts.py -# script using the template file and common include files in scripts/includes/*.sh. -# -# DO NOT EDIT MANUALLY. -# -# Please edit corresponding template file and include files. -""".strip() - - -def main(): - with open(COMMON_INCLUDE_PATH, 'r') as fp: - common_script_content = fp.read() - - with open(RHEL_INCLUDE_PATH, 'r') as fp: - rhel_common_script_content = fp.read() - - for script_filename in SCRIPT_FILES: - script_file_path = os.path.join(SCRIPTS_PATH, script_filename) - template_file_path = script_file_path.replace('.sh', '.template.sh') - - print('Generating script file "%s" -> "%s"' % (template_file_path, script_file_path)) - - with open(template_file_path, 'r') as fp: - template_content = fp.read() - - result = '' - result += HEADER_WARNING - result += '\n\n' - result += template_content - - # Add in content from includes/ files - result = result.replace('# include:includes/common.sh', common_script_content) - result = result.replace('# include:includes/rhel.sh', rhel_common_script_content) - - with open(script_file_path, 'w') as fp: - fp.write(result) - - print('File "%s" has been generated.' % (script_file_path)) - - -if __name__ == '__main__': - main() diff --git a/tools/generate_install_script.json b/tools/generate_install_script.json new file mode 100644 index 00000000..cf2d326c --- /dev/null +++ b/tools/generate_install_script.json @@ -0,0 +1,33 @@ +[ + { + "comment": "this is here for backward compatibility with install script. To be removed when the install script stops using master branch and uses st2 version branch.", + "id": "ubuntu", + "version_id": "22.04", + "script_filename": "st2bootstrap-deb.sh", + "pkg_mgr": "apt" + }, + { + "id": "ubuntu", + "version_id": "20.04", + "script_filename": "st2bootstrap-focal.sh", + "pkg_mgr": "apt" + }, + { + "id": "ubuntu", + "version_id": "22.04", + "script_filename": "st2bootstrap-jammy.sh", + "pkg_mgr": "apt" + }, + { + "id": "rocky", + "version_id": "8", + "script_filename": "st2bootstrap-el8.sh", + "pkg_mgr": "dnf" + }, + { + "id": "rocky", + "version_id": "9", + "script_filename": "st2bootstrap-el9.sh", + "pkg_mgr": "dnf" + } +] diff --git a/tools/generate_install_script.py b/tools/generate_install_script.py new file mode 100755 index 00000000..4566cf97 --- /dev/null +++ b/tools/generate_install_script.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Generate st2bootstrap install scripts based on a set of supported operating systems. + +A bootstrap script is generated per operating system based on a central jinja template +that includes other templates based on information provided in the operating system +data file `generate_install_script.json`. +""" +import os +import json +from jinja2 import Environment, FileSystemLoader + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +SCRIPTS_PATH = os.path.abspath(os.path.join(BASE_DIR, "../scripts")) +TEMPLATE_PATH = os.path.abspath(os.path.join(BASE_DIR, "../templates")) +TEMPLATE_FILE = "st2bootstrap.jinja" +DATA_FILE = os.path.abspath(os.path.join(BASE_DIR, "generate_install_script.json")) + + +def load_json(filename): + """ + Load JSON from file and return the deserialised content. + """ + with open(filename, "r", encoding="utf-8") as f: + return json.load(f) + + +def fetch_template(template="st2bootstrap.jinja", template_dir="."): + """ + Load Jinja template and return the template object. + """ + file_loader = FileSystemLoader(template_dir) + env = Environment(loader=file_loader) + return env.get_template(template) + + +def render_document(template, data, filename, backup_file=False): + """ + Given the template object and data, render the template and write to the filename. + """ + if backup_file: + if not os.path.exists(f"{filename}.bak") and os.path.exists(f"{filename}"): + os.replace(f"{filename}", f"{filename}.bak") + with open(filename, "w", encoding="utf-8") as f: + f.write(template.render(data)) + + +def main(): + """ + Entry point. + """ + # $ID and $VERSION_ID are sourced from /etc/os-release + # In the case where VERSION_ID contains major.minor id, it's possible + # to list the key with only the major id and have any minor id match + # against it to use the same template across a major version of the distribuion. + # Adding a key with the required data will produce a bootstrap script + # for the OS. + data = load_json(DATA_FILE) + + for os_data in data: + os_id = os_data["id"] + os_version_id = os_data["version_id"] + + script_filename = os_data["script_filename"] + script_abs_filename = os.path.join(SCRIPTS_PATH, script_filename) + + print(f"Generating script file '{script_filename}' for {os_id} {os_version_id}") + template = fetch_template(TEMPLATE_FILE, TEMPLATE_PATH) + render_document(template, os_data, script_abs_filename) + + +if __name__ == "__main__": + main()