PLAT-9091 - Updated rke, rancher, tools

.circleci/config.yml

@@ -1,21 +1,30 @@
+# CircleCI 2.1 configuration file
+# Check https://circleci.com/docs/2.0/sample-config/ for more details
+#
 version: 2.1
 
 executors:
+  trivy:
+    docker:
+      - image: aquasec/trivy:0.60.0
+    environment:
+      ENV_FILE: /tmp/workspace/.env
+      WORKSPACE: /tmp/workspace
   default:
     docker:
-      - image: cimg/python:3.9
+      - image: cimg/python:3.12
     environment:
-      TERRAFORM_VERSION: 1.2.4
-      ANSIBLE_VERSION: 2.13.3
+      TERRAFORM_VERSION: 1.11.2
+      ANSIBLE_VERSION: 2.18.3
       JMESPATH_VERSION: 1.0.1
       RANCHER_PASSWORD: thisisanewadminpassword
 
 orbs:
-  aws-cli: circleci/aws-cli@3.1.1
+  aws-cli: circleci/aws-cli@5.2.0
 
 commands:
   install_ansible:
-    description: "Install Ansible"
+    description: Install Ansible
     steps:
       - run:
           name: Install Ansible & Dependencies
@@ -34,7 +43,7 @@ commands:
             sudo unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin
 
   launch_e2e:
-    description: "Launch an e2e test using an AWS Lightsail VPS"
+    description: Launch an e2e test using an AWS Lightsail VPS
     parameters:
       blueprint_id:
         type: string
@@ -76,7 +85,7 @@ commands:
 
 jobs:
   lint:
-    description: "Check Ansible Syntax"
+    description: Check Ansible Syntax
     executor: default
     steps:
       - checkout
@@ -93,45 +102,47 @@ jobs:
       - run:
           name: Check Ansible Syntax
           command: ansible-playbook -i '127.0.0.1,' ansible/prod.yml --syntax-check
-  e2e_centos:
-    executor: default
-    steps:
-    - launch_e2e:
-        ssh_user: centos
-        blueprint_id: centos_7_1901_01
-  e2e_ubuntu:
-    executor: default
+  scan:
+    executor: trivy
     steps:
-    - launch_e2e:
-        ssh_user: ubuntu
-        blueprint_id: ubuntu_16_04_2
-  e2e_ubuntu18:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Install trivy
+          command: |
+            apk add --update-cache --upgrade curl
+            curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
+      - run:
+          name: Scan filesystem
+          command: |
+            trivy fs --include-non-failures --misconfig-scanners terraform \
+            --exit-code 0 --no-progress \
+            --scanners vuln,secret,config --severity CRITICAL,HIGH,MEDIUM,LOW \
+            --output "trivy-results.json" --format json --ignore-unfixed .
+      - store_artifacts:
+          path: trivy-results.json
+          destination: trivy_output
+  e2e_ubuntu20:
     executor: default
     steps:
     - launch_e2e:
         ssh_user: ubuntu
-        blueprint_id: ubuntu_18_04
-  e2e_ubuntu20:
+        blueprint_id: ubuntu_20_04
+  e2e_ubuntu22:
     executor: default
     steps:
     - launch_e2e:
         ssh_user: ubuntu
-        blueprint_id: ubuntu_20_04
+        blueprint_id: ubuntu_22_04
 
 workflows:
-  version: 2
-  ansible_integrations:
+  Lint and Scan:
     jobs:
     - lint
-    # - e2e_centos:
-    #     requires:
-    #     - lint
-    # - e2e_ubuntu:
-    #     requires:
-    #     - lint
-    # - e2e_ubuntu18:
+    - scan
+    # - e2e_ubuntu20:
     #     requires:
     #     - lint
-    # - e2e_ubuntu20:
+    # - e2e_ubuntu22:
     #     requires:
     #     - lint

.github/CODEOWNERS

@@ -1 +1 @@
-* @dominodatalab/platform
+* @dominodatalab/eng-distributions

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: no-commit-to-branch
+      - id: check-case-conflict
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.97.4
+    hooks:
+      - id: terraform_validate
+        args:
+          - '--hook-config=--retry-once-with-cleanup=true'
+      - id: terraform_providers_lock
+        args:
+          - --tf-init-args=-upgrade
+      - id: terraform_docs
+        args:
+          - '--args=--lockfile=false'
+          - '--hook-config=--path-to-file=README.md'
+          - '--hook-config=--add-to-existing-file=true'
+          - '--hook-config=--create-file-if-not-exist=true'
+      - id: terraform_fmt
+      - id: terraform_tflint
+        args:
+          - '--args=--config=__GIT_WORKING_DIR__/.tflint.hcl'

.tflint.hcl

@@ -0,0 +1,6 @@
+plugin "aws" {
+  enabled = true
+  deep_check = false
+  version = "0.38.0"
+  source  = "github.com/terraform-linters/tflint-ruleset-aws"
+}

README.md

@@ -1,22 +1,22 @@
 # RanchHand
 
 [![Release](https://img.shields.io/github/release/dominodatalab/ranchhand.svg)](https://github.com/dominodatalab/ranchhand/releases/latest)
-[![CircleCI](https://img.shields.io/circleci/project/github/dominodatalab/ranchhand/master.svg)](https://circleci.com/gh/dominodatalab/ranchhand)
+[![CircleCI](https://dl.circleci.com/status-badge/img/gh/dominodatalab/ranchhand/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/dominodatalab/ranchhand/tree/develop)
 
 Deploy Rancher in HA mode onto existing hardware.
 
 ## Design
 
-This tool aims to automate the steps listed in Rancher's official [HA Install][] documentation in a reproducable manner. It also enforces many of the recommendations given inside Rancher's [hardening guide][].
+This tool aims to automate the steps listed in Rancher's official [HA Install](https://ranchermanager.docs.rancher.com/reference-guides/rancher-manager-architecture/architecture-recommendations#why-ha-is-better-for-rancher-in-production) documentation in a reproducable manner. It also enforces many of the recommendations given inside Rancher's [hardening guide](https://releases.rancher.com/documents/security/latest/Rancher_Hardening_Guide.pdf).
 
 ## Usage
 
 1. Ensure that the Rancher version is >= 2.6. 
-1. Download the [latest release][] from GitHub.
+1. Download the [latest release](https://github.com/dominodatalab/RanchHand/releases/latest) from GitHub.
 1. [Install Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) (version >=2.8) locally
-1.  (optional) To update the Rancher default password, set the `RANCHER_PASSWORD` environment variable:
+1. (_optional_) To update the Rancher default password, set the `RANCHER_PASSWORD` environment variable:
      `export RANCHER_PASSWORD=<new password>`
-    1.    (required) Configure ansible for proper output:
+    1. (**required**) Configure ansible for proper output:
             `export ANSIBLE_COW_SELECTION=random`
 1. Execute `ansible-galaxy install -r ansible/requirements.yml` to install dependencies
 1. Execute `ansible-playbook -i '1.2.4.5,...,10.20.30.40,' --private-key=~/.ssh/id_rsa --user=ubuntu ansible/prod.yml --diff --check` to perform a dry run of all the changes.
@@ -25,19 +25,19 @@ This tool aims to automate the steps listed in Rancher's official [HA Install][]
 
 This example shows a manual run of the production playbook (prod.yml) from a local machine imaging a cluster behind a bastion/proxy server.
 
-```
+```shell
 ansible-playbook -i '10.0.1.6,10.0.1.51,10.0.1.94,' --private-key=/Users/myhost/.ssh/id_rsa --user=ubuntu --ssh-common-args='-o StrictHostKeyChecking=no -o StrictHostKeyChecking=no -o ProxyCommand="ssh -o StrictHostKeyChecking=no -W %h:%p -q [email protected]"' ansible/prod.yml --diff
 ```
 
-In the example above, only the bastion server, 54.190.1.95, is publicly accessible. However, including the Terraform module should be sufficient for most users.
+In the example above, only the bastion server, `54.190.1.95`, is publicly accessible. However, including the Terraform module should be sufficient for most users.
 
 ## Terraform
 
-Using the Terraform module, you can leverage Ranchhand to create a Rancher cluster on a specific set of nodes.
+Using the Terraform module, you can leverage RanchHand to create a Rancher cluster on a specific set of nodes.
 
 ```hcl
-module "ranchhand" {
-  source = "github.com/dominodatalab/ranchhand"
+module "RanchHand" {
+  source = "github.com/dominodatalab/RanchHand"
 
   node_ips         = ["..."]
   working_dir      = "..."
@@ -55,20 +55,19 @@ module "ranchhand" {
 
 Please submit any feature enhancements, bug fixes, or ideas via pull requests or issues.  If you need to test local changes e2e, you can do so using Vagrant and Virtualbox. Here are the recommended steps:
 
-1. Make sure you have Vagrant and VirtualBox installed.
-
-    `brew cask install vagrant virtualbox`
+1. Make sure you have Vagrant and VirtualBox installed by running `brew install --cask vagrant virtualbox`
 
-1. Create one or more VMs. For convenience, a pre-configured [Vagrantfile][] is available.
+1. Create one or more VMs. For convenience, a pre-configured [Vagrantfile](./test/Vagrantfile) is available.
 
-    ```
+    ```shell
     cd test/
-    NODE_COUNT=N NODE_DISTRO="ubuntu_xenial|ubuntu_bionic|centos|rhel" vagrant up
+    # only specify one distro below
+    NODE_COUNT=N NODE_DISTRO="ubuntu_jammy|ubuntu_focal|rhel" vagrant up
     ```
 
-1. Use `ansible` to launch a Ranchhand run against your VM(s) and verify your changes.
+1. Use `ansible` to launch a local RanchHand run against your VM(s) and verify your changes.
 
-    ```
+    ```shell
     ansible-galaxy install -r ansible/requirements.yml
     ansible-playbook -i '192.168.50.10,' \
       --private-key=~/.ssh/id_rsa \
@@ -82,6 +81,8 @@ Please submit any feature enhancements, bug fixes, or ideas via pull requests or
 
     _Note the trailing comma (",") in the host/ip list._
 
+1. When testing is complete, vagrant boxes can be destroyed with `vagrant destroy`
+
 ### Ansible References
 
 Here are some helpful Ansible references for getting started with Ansible.
@@ -95,10 +96,3 @@ Here are some helpful Ansible references for getting started with Ansible.
 ## Contribute
 
 Contributions are always welcome! Please submit any questions, bugs or changes via an issue or PR. Thank you.
-
-[vagrantfile]: test/Vagrantfile
-
-[rke]: https://github.com/rancher/rke
-[ha install]: https://rancher.com/docs/rancher/v2.x/en/installation/ha/
-[hardening guide]: https://releases.rancher.com/documents/security/latest/Rancher_Hardening_Guide.pdf
-[latest release]: https://github.com/dominodatalab/ranchhand/releases/latest

ansible/ansible.cfg

@@ -1,2 +1,2 @@
 [ssh_connection]
-pipelining = True
+pipelining = true

ansible/prod.yml

@@ -12,7 +12,7 @@
   vars_prompt:
     - name: cert_names
       prompt: CA 'comma separated string' of values prefixed by their options. (i.e., email, URI, DNS, RID, IP, dirName, otherName and the ones specific to your CA)
-      private: no
+      private: false
 
   pre_tasks:
     - name: Create local output directory

ansible/roles/docker/defaults/main.yml

@@ -1,9 +1,8 @@
 ---
 docker_version:
-  xenial: 5:18.09.8~3-0~ubuntu-xenial
   focal: 5:19.03.12~3-0~ubuntu-focal
-  bionic: 5:18.09.9~3-0~ubuntu-bionic
   jammy: 5:24.0.9-1~ubuntu.22.04~jammy
+  noble: 5:26.0.1-1~ubuntu.24.04~noble
 docker_ce_yum_version: 18.09.6-3.el7
 containerd_yum_version: 1.2.10-3.2.el7
 container_selinux_yum_version: 2.107-3.el7

ansible/roles/docker/handlers/main.yml

@@ -1,6 +1,6 @@
 ---
 - name: restart docker
-  become: yes
+  become: true
   systemd:
     state: restarted
     name: docker

ansible/roles/docker/tasks/main.yml

@@ -8,13 +8,13 @@
   when: ansible_os_family == "RedHat"
 
 - name: Add user to docker group
-  become: yes
+  become: true
   user:
     name: "{{ ansible_user_id }}"
     group: docker
 
 - name: Configure Docker daemon options
-  become: yes
+  become: true
   template:
     src: "daemon.json.j2"
     dest: "/etc/docker/daemon.json"

ansible/roles/docker/tasks/redhat.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install Docker & Dependencies
-  become: yes
+  become: true
   yum:
     name: "{{ packages }}"
   vars:
@@ -11,8 +11,8 @@
       - "{{ docker_ce_repo_root }}/container-selinux-{{ container_selinux_yum_version }}.noarch.rpm"
 
 - name: Enable and Start Docker
-  become: yes
+  become: true
   systemd:
     state: started
     name: docker
-    enabled: yes
+    enabled: true

ansible/roles/docker/vars/main.yml

@@ -1,2 +1,2 @@
 ---
-dom_debug: False
+dom_debug: false

ansible/roles/newrelic/tasks/newrelic.yml

@@ -1,6 +1,6 @@
 - name: Login to GCR repo
   shell: echo {{ helm['password'] }} | base64 -d | helm registry login {{ helm['host'] }} --username {{ helm['user'] }} --password-stdin
-  run_once: yes
+  run_once: true
   no_log: true
 
 - name: Probe bitnami repo
@@ -14,13 +14,13 @@
 - name: Probe Newrelic Namespace
   command: kubectl get ns {{ newrelic['namespace'] }}
   changed_when: false
-  ignore_errors: yes
-  run_once: yes
+  ignore_errors: true
+  run_once: true
   register: rancher_monitoring
 
 - name: Create Newrelic namespace
   command: kubectl create namespace {{ newrelic['namespace'] }}
-  run_once: yes
+  run_once: true
   when: rancher_monitoring is failed
 
 - name: Install Kube State Metrics

ansible/roles/newrelic/vars/main.yml

@@ -1,2 +1,2 @@
 ---
-dom_debug: False
+dom_debug: false

ansible/roles/rancher/tasks/cert-manager.yml

@@ -2,8 +2,8 @@
 - name: Probe cert-manager Namespace
   command: kubectl get ns cert-manager
   changed_when: false
-  ignore_errors: yes
-  run_once: yes
+  ignore_errors: true
+  run_once: true
   register: certmanager_ns
 
 - name: Create cert-manager Namespace

ansible/roles/rancher/tasks/rancher.yml

@@ -11,7 +11,7 @@
     name: rancher
     namespace: cattle-system
     state: present
-    update_repo_cache: yes
+    update_repo_cache: true
     chart_ref: rancher-stable/rancher
     chart_version: "{{ rancher_version }}"
     atomic: true
@@ -37,7 +37,7 @@
   uri:
     url: https://localhost/ping
     method: GET
-    validate_certs: no
+    validate_certs: false
   register: result
   until: result.status == 200
   retries: 10