collectd + Graphite + Grafana + Zabbix + nginx/certbot with SSL cert …

…autorenewal on docker

README.md

@@ -21,12 +21,13 @@ Build your own team IT infrastructure with ~blackjack~ encrypted private cloud,
 * Software distribution server [storage and docker registry](roles/distribution_hub) based on [Nexus Repository Manager 3](https://github.com/sonatype/docker-nexus3)
 behind [nginx for SSL termination](roles/nginx)
 * Media [server](playbooks/openmediavault.yml) from [openmediavault.org](https://www.openmediavault.org)
-* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files) and [collectd](roles/collectd_beacon) 
+* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files/dockprom)
 based on [dockprom](https://github.com/stefanprodan/dockprom)
+* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [collectd](roles/collectd_beacon) and [Graphite + Grafana + Zabbix + nginx/certbot with SSL cert autorenewal](roles/monitoring_hub/files/monitoring_hub)
 * BIND DNS server bundled with the Webmin UI based on [sameersbn's docker-bind](https://github.com/sameersbn/docker-bind)
 
 ###### Privacy
-* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/)
+* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn.yml) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/)
 
 ###### Security
 * [SSH users ACL and management](tests/test_deploy_users.sh) with public ssh keys and common sudoer user
@@ -58,7 +59,7 @@ ansible-playbook playbooks/openvpn-server.yml
 
 See example test [test_deploy_openvpn.sh](tests/test_deploy_openvpn.sh)
 
-1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn).
+1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn.yml).
 Client may reserve static VPN IP or dynamic otherwise.
 
 
@@ -82,7 +83,7 @@ ansible-playbook -i environments/test/inventory playbooks/openvpn-client.yml
 ```
 
 ##### Revoke VPN key
-1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory).
+1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory)
 2. Update OpenVPN server:
 ```bash
 ansible-playbook -i environments/test/inventory playbooks/openvpn-server.yml --limit openvpn-server

environments/test/group_vars/all.yml

@@ -1,4 +1,4 @@
-monitoring_grafana_public_root_url: http://192.168.10.101/
+monitoring_graphite_host: 192.168.10.101
 
 # extract subdir name from environment name (e.g. environments/test -> vpnkeys/test)
 vpnkeys_dir: "../.vpnkeys/{{ inventory_dir.split('/')[-1] }}/"

roles/collectd_beacon/tasks/main.yml

@@ -15,10 +15,6 @@
     name: hddtemp
     state: restarted
 
-- name: install collectd-cuda
-  shell: |
-    pip3 install collectd-cuda
-
 - name: add hostname to /etc/hosts (FQDN lookup required for collectd to start)
   lineinfile:
     dest: /etc/hosts
@@ -31,7 +27,7 @@
     name: collectd
     state: present
 
-- name: configure /etc/collectd/collectd.conf'
+- name: configure /etc/collectd/collectd.conf to send metrics to '{{monitoring_graphite_host}}'
   template:
     src: templates/collectd.conf.j2
     dest: /etc/collectd/collectd.conf

roles/collectd_beacon/templates/collectd.conf.j2

@@ -1243,7 +1243,7 @@ LoadPlugin write_prometheus
 
 <Plugin write_graphite>
 	<Node "example">
-		Host "monitoring_graphite_host"
+		Host "{{monitoring_graphite_host}}"
 		Port "2003"
 		Protocol "tcp"
 		LogSendErrors true

roles/monitoring_hub/files/monitoring_hub/docker-compose.yml

@@ -0,0 +1,135 @@
+version: '2'
+services:
+
+  nginx:
+    restart: unless-stopped
+    image: nginx:1.20
+    command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & /docker-entrypoint.sh nginx -g \"daemon off;\" '"
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./nginx/conf.d:/etc/nginx/conf.d
+      - ./nginx/nginx-auth.htpasswd:/etc/nginx/nginx-auth.htpasswd
+      # certbox
+      - ./nginx/certbot/conf:/etc/letsencrypt
+      - ./nginx/certbot/www:/var/www/certbot
+
+  certbot:
+    image: certbot/certbot
+    restart: unless-stopped
+    volumes:
+      - ./nginx/certbot/conf:/etc/letsencrypt
+      - ./nginx/certbot/www:/var/www/certbot
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
+
+  grafana:
+    restart: unless-stopped
+    image: grafana/grafana:7.4.5
+    ports:
+      - 83:3000
+    volumes:
+      - ./grafana:/var/lib/grafana
+      #- ./custom.ini:/opt/grafana/conf/custom.ini
+    # chown -R 472:472 ./grafana
+    user: "root"
+    environment:
+      GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource,alexanderzobnin-zabbix-app,vonage-status-panel
+      GF_SECURITY_ADMIN_USER: ${ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SERVER_ROOT_URL: https://grafana.orgbackbone.org
+      # if behind nginx with basicauth with different passwords
+      GF_AUTH_BASIC_ENABLED: "false"
+      GF_RENDERING_SERVER_URL: http://renderer:8081/render
+      GF_RENDERING_CALLBACK_URL: http://grafana:3000/
+      GF_LOG_FILTERS: rendering:debug
+
+  renderer:
+    image: grafana/grafana-image-renderer:2.0.1
+    ports:
+      - 8081
+    environment:
+      ENABLE_METRICS: 'true'
+
+  graphite:
+    image: graphiteapp/graphite-statsd:1.1.8-1
+    hostname: graphite
+    ports:
+      - 81:80       # nginx-gunicorn-graphite
+      #- 8080:8080   # gunicorn graphite 
+      - 2003-2004:2003-2004
+      - 2023-2024:2023-2024
+      - 8125:8125/udp
+      - 8126:8126
+    volumes:
+      - ./graphite/storage:/opt/graphite/storage
+    environment:
+      - COLLECTD=1
+      - REDIS_TAGDB=1
+
+  postgresql:
+    image: postgres:12
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: zabbix
+      POSTGRES_PASSWORD: zabbix
+      POSTGRES_DB: zabbixdb
+    volumes:
+      - ./postgresql/data:/var/lib/postgresql/data
+
+  zabbix-server:
+    image: zabbix/zabbix-server-pgsql:ubuntu-5.2-latest
+    restart: unless-stopped
+    hostname: zabbix-server-container
+    environment:
+      DB_SERVER_HOST: postgresql
+      POSTGRES_USER: zabbix
+      POSTGRES_PASSWORD: zabbix
+      POSTGRES_DB: zabbixdb
+      ZBX_HISTORYSTORAGETYPES: log,text
+      ZBX_DEBUGLEVEL: 1
+      ZBX_HOUSEKEEPINGFREQUENCY: 1
+      ZBX_MAXHOUSEKEEPERDELETE: 5000
+      ZBX_TRAPPERIMEOUT: 300
+    depends_on:
+      - postgresql
+    volumes:
+      - ./zabbix/alertscripts:/usr/lib/zabbix/alertscripts
+      - ./zabbix/externalscripts:/usr/lib/zabbix/externalscripts:z
+      - ./zabbix/modules:/var/lib/zabbix/modules:z
+      - ./zabbix/enc:/var/lib/zabbix/enc:z
+      - ./zabbix/ssh_keys:/var/lib/zabbix/ssh_keys:z
+      - ./zabbix/ssl/certs:/var/lib/zabbix/ssl/certs:z
+      - ./zabbix/ssl/keys:/var/lib/zabbix/ssl/keys:z
+      - ./zabbix/ssl/ssl_ca:/var/lib/zabbix/ssl/ssl_ca:z
+      - ./zabbix/snmptraps:/var/lib/zabbix/snmptraps:z
+      - ./zabbix/mibs:/var/lib/zabbix/mibs:z
+      - ./zabbix/export:/var/lib/zabbix/export:z
+    ports:
+      - 10051:10051
+
+  zabbix-web:
+    image: zabbix/zabbix-web-nginx-pgsql:ubuntu-5.2-latest
+    restart: unless-stopped
+    environment:
+      DB_SERVER_HOST: postgresql
+      POSTGRES_USER: zabbix
+      POSTGRES_PASSWORD: zabbix
+      POSTGRES_DB: zabbixdb
+      ZBX_SERVER_HOST: zabbix-server  # Zabbix related and Php variables
+      ZBX_POSTMAXSIZE: 64M
+      ZBX_MAXEXECUTIONTIME: 500
+    depends_on:
+      - postgresql
+      - zabbix-server
+
+  zabbix-agent:
+    image: zabbix/zabbix-agent:ubuntu-5.2-latest
+    # privileged: true # access mode for allowing resource access
+    hostname: zabbix-agent-container
+    restart: unless-stopped
+    environment:
+      - ZBX_SERVER_HOST=zabbix-server
+    ports:
+      - 10050:10050

roles/monitoring_hub/files/monitoring_hub/init-letsencrypt.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# based on https://github.com/wmnnd/nginx-certbot
+# see https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
+#
+set -xe
+
+
+if ! [ -x "$(command -v docker-compose)" ]; then
+  echo 'Error: docker-compose is not installed.' >&2
+  exit 1
+fi
+
+domains=(grafana.orgbackbone.org zabbix.orgbackbone.org)
+rsa_key_size=4096
+data_path="./nginx/certbot"
+email="[email protected]" # Adding a valid address is strongly recommended
+staging=0 # Set to 1 if you're testing your setup to avoid hitting request limits
+
+if [ -d "$data_path" ]; then
+  read -p "Existing data found for $domains. Continue and replace existing certificate? (y/N) " decision
+  if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then
+    exit
+  fi
+fi
+
+
+if [ ! -e "$data_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$data_path/conf/ssl-dhparams.pem" ]; then
+  echo "### Downloading recommended TLS parameters ..."
+  mkdir -p "$data_path/conf"
+  curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$data_path/conf/options-ssl-nginx.conf"
+  curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$data_path/conf/ssl-dhparams.pem"
+  echo
+fi
+
+echo "### Creating dummy certificate for $domains ..."
+path="/etc/letsencrypt/live/$domains"
+mkdir -p "$data_path/conf/live/$domains"
+docker-compose run --rm --entrypoint "\
+  openssl req -x509 -nodes -newkey rsa:$rsa_key_size -days 1\
+    -keyout '$path/privkey.pem' \
+    -out '$path/fullchain.pem' \
+    -subj '/CN=localhost'" certbot
+echo
+
+
+echo "### Starting nginx ..."
+docker-compose up --force-recreate -d nginx
+echo
+
+echo "### Deleting dummy certificate for $domains ..."
+docker-compose run --rm --entrypoint "\
+  rm -Rf /etc/letsencrypt/live/$domains && \
+  rm -Rf /etc/letsencrypt/archive/$domains && \
+  rm -Rf /etc/letsencrypt/renewal/$domains.conf" certbot
+echo
+
+
+echo "### Requesting Let's Encrypt certificate for $domains ..."
+#Join $domains to -d args
+domain_args=""
+for domain in "${domains[@]}"; do
+  domain_args="$domain_args -d $domain"
+done
+
+# Select appropriate email arg
+case "$email" in
+  "") email_arg="--register-unsafely-without-email" ;;
+  *) email_arg="--email $email" ;;
+esac
+
+# Enable staging mode if needed
+if [ $staging != "0" ]; then staging_arg="--staging"; fi
+
+docker-compose run --rm --entrypoint "\
+  certbot certonly --webroot -w /var/www/certbot \
+    $staging_arg \
+    $email_arg \
+    $domain_args \
+    --rsa-key-size $rsa_key_size \
+    --agree-tos \
+    --no-eff-email \
+    --force-renewal" certbot
+echo
+
+echo "### Reloading nginx ..."
+docker-compose exec nginx nginx -s reload

roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/certbot-challenge.conf

@@ -0,0 +1,10 @@
+server {
+  listen 80 default_server;
+  listen [::]:80 default_server;
+
+  location /.well-known/acme-challenge/ { root /var/www/certbot;  }
+  location / { return 301 https://$host$request_uri; }
+}
+
+# add docker's embedded DNS 127.0.0.11 to route docker services with container name
+resolver 127.0.0.11 127.0.0.1 [::1]:5353 valid=5s;

roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/grafana.conf

@@ -0,0 +1,26 @@
+server {
+  listen 443 ssl;
+  server_name grafana.orgbackbone.org;
+
+  ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem;
+
+  include /etc/letsencrypt/options-ssl-nginx.conf;
+  ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+
+  auth_basic "";
+  auth_basic_user_file /etc/nginx/nginx-auth.htpasswd;
+
+  location / {
+    proxy_pass              http://grafana:3000;
+    proxy_set_header        Host $host;
+    proxy_set_header        Referer "";
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        Upgrade $http_upgrade;
+    proxy_set_header        Connection "upgrade";
+    proxy_http_version      1.1;
+  }
+}
+
+

roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/zabbix.conf

@@ -0,0 +1,26 @@
+server {
+  listen 443 ssl;
+  server_name zabbix.orgbackbone.org;
+
+  ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem;
+
+  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+  ssl_ciphers HIGH:!aNULL:!MD5;
+
+  auth_basic "";
+  auth_basic_user_file /etc/nginx/nginx-auth.htpasswd;
+
+  location / {
+    proxy_pass              http://zabbix-web:8080/;
+    proxy_set_header        Host $host;
+    proxy_set_header        Referer "";
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        Upgrade $http_upgrade;
+    proxy_set_header        Connection "upgrade";
+    proxy_http_version      1.1;
+  }
+}
+
+

roles/monitoring_hub/files/monitoring_hub/nginx/nginx-auth.htpasswd

@@ -0,0 +1,3 @@
+USER:um2j1VlJgPYvw
+
+

roles/monitoring_hub/tasks/main.yml

@@ -1,3 +1,4 @@
+# just dockprom installer
 - name: deploy dockprom
   copy:
     src: files/dockprom
@@ -15,3 +16,28 @@
   args:
     chdir: /opt/dockprom
     warn: false
+
+# monitoring_hub: graphite + grafana + zabbix
+- name: copy monitoring_hub configs
+  copy:
+    src: files/monitoring_hub
+    dest: /opt
+    owner: dev
+
+- name: generate nginx auth
+  shell: |
+    printf "USER:$(openssl passwd -crypt PASSWORD)\n" >> nginx/nginx-auth.htpasswd
+  args:
+    chdir: /opt/monitoring_hub
+
+- name: init-letsencrypt.sh
+  shell: |
+    bash init-letsencrypt.sh
+  args:
+    chdir: /opt/monitoring_hub
+
+- name: build and start docker containers
+  shell: |
+    docker-compose up -d
+  args:
+    chdir: /opt/monitoring_hub