diff --git a/README.md b/README.md index 6c17dce..711eeb9 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,13 @@ Build your own team IT infrastructure with ~blackjack~ encrypted private cloud, * Software distribution server [storage and docker registry](roles/distribution_hub) based on [Nexus Repository Manager 3](https://github.com/sonatype/docker-nexus3) behind [nginx for SSL termination](roles/nginx) * Media [server](playbooks/openmediavault.yml) from [openmediavault.org](https://www.openmediavault.org) -* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files) and [collectd](roles/collectd_beacon) +* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [Grafana + Prometheus](roles/monitoring_hub/files/dockprom) based on [dockprom](https://github.com/stefanprodan/dockprom) +* [Infrastructure monitoring & alerting](tests/test_deploy_monitoring.sh) with [collectd](roles/collectd_beacon) and [Graphite + Grafana + Zabbix + nginx/certbot with SSL cert autorenewal](roles/monitoring_hub/files/monitoring_hub) * BIND DNS server bundled with the Webmin UI based on [sameersbn's docker-bind](https://github.com/sameersbn/docker-bind) ###### Privacy -* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/) +* [OpenVPN](tests/test_deploy_openvpn.sh) and [keys management](environments/test/group_vars/openvpn.yml) based on [Stouts.openvpn ansible role](https://github.com/Stouts/Stouts.openvpn/) ###### Security * [SSH users ACL and management](tests/test_deploy_users.sh) with public ssh keys and common sudoer user @@ -58,7 +59,7 @@ ansible-playbook playbooks/openvpn-server.yml See example test [test_deploy_openvpn.sh](tests/test_deploy_openvpn.sh) -1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn). +1. Add `username` entry into list of **openvpn_clients_active** in [environments/test/group_vars/openvpn](environments/test/group_vars/openvpn.yml). Client may reserve static VPN IP or dynamic otherwise. @@ -82,7 +83,7 @@ ansible-playbook -i environments/test/inventory playbooks/openvpn-client.yml ``` ##### Revoke VPN key -1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory). +1. Add client's name into `openvpn_clients_revoke` blacklist of [environments/test/inventory](environments/test/inventory) 2. Update OpenVPN server: ```bash ansible-playbook -i environments/test/inventory playbooks/openvpn-server.yml --limit openvpn-server diff --git a/environments/test/group_vars/all.yml b/environments/test/group_vars/all.yml index ee9f28a..661d8f2 100644 --- a/environments/test/group_vars/all.yml +++ b/environments/test/group_vars/all.yml @@ -1,4 +1,4 @@ -monitoring_grafana_public_root_url: http://192.168.10.101/ +monitoring_graphite_host: 192.168.10.101 # extract subdir name from environment name (e.g. environments/test -> vpnkeys/test) vpnkeys_dir: "../.vpnkeys/{{ inventory_dir.split('/')[-1] }}/" diff --git a/roles/collectd_beacon/tasks/main.yml b/roles/collectd_beacon/tasks/main.yml index a285211..28b9d9a 100644 --- a/roles/collectd_beacon/tasks/main.yml +++ b/roles/collectd_beacon/tasks/main.yml @@ -15,10 +15,6 @@ name: hddtemp state: restarted -- name: install collectd-cuda - shell: | - pip3 install collectd-cuda - - name: add hostname to /etc/hosts (FQDN lookup required for collectd to start) lineinfile: dest: /etc/hosts @@ -31,7 +27,7 @@ name: collectd state: present -- name: configure /etc/collectd/collectd.conf' +- name: configure /etc/collectd/collectd.conf to send metrics to '{{monitoring_graphite_host}}' template: src: templates/collectd.conf.j2 dest: /etc/collectd/collectd.conf diff --git a/roles/collectd_beacon/templates/collectd.conf.j2 b/roles/collectd_beacon/templates/collectd.conf.j2 index c50717b..20ea892 100644 --- a/roles/collectd_beacon/templates/collectd.conf.j2 +++ b/roles/collectd_beacon/templates/collectd.conf.j2 @@ -1243,7 +1243,7 @@ LoadPlugin write_prometheus - Host "monitoring_graphite_host" + Host "{{monitoring_graphite_host}}" Port "2003" Protocol "tcp" LogSendErrors true diff --git a/roles/monitoring_hub/files/monitoring_hub/docker-compose.yml b/roles/monitoring_hub/files/monitoring_hub/docker-compose.yml new file mode 100644 index 0000000..3f230ef --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/docker-compose.yml @@ -0,0 +1,135 @@ +version: '2' +services: + + nginx: + restart: unless-stopped + image: nginx:1.20 + command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & /docker-entrypoint.sh nginx -g \"daemon off;\" '" + ports: + - 80:80 + - 443:443 + volumes: + - ./nginx/conf.d:/etc/nginx/conf.d + - ./nginx/nginx-auth.htpasswd:/etc/nginx/nginx-auth.htpasswd + # certbox + - ./nginx/certbot/conf:/etc/letsencrypt + - ./nginx/certbot/www:/var/www/certbot + + certbot: + image: certbot/certbot + restart: unless-stopped + volumes: + - ./nginx/certbot/conf:/etc/letsencrypt + - ./nginx/certbot/www:/var/www/certbot + entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" + + grafana: + restart: unless-stopped + image: grafana/grafana:7.4.5 + ports: + - 83:3000 + volumes: + - ./grafana:/var/lib/grafana + #- ./custom.ini:/opt/grafana/conf/custom.ini + # chown -R 472:472 ./grafana + user: "root" + environment: + GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource,alexanderzobnin-zabbix-app,vonage-status-panel + GF_SECURITY_ADMIN_USER: ${ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_SERVER_ROOT_URL: https://grafana.orgbackbone.org + # if behind nginx with basicauth with different passwords + GF_AUTH_BASIC_ENABLED: "false" + GF_RENDERING_SERVER_URL: http://renderer:8081/render + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + + renderer: + image: grafana/grafana-image-renderer:2.0.1 + ports: + - 8081 + environment: + ENABLE_METRICS: 'true' + + graphite: + image: graphiteapp/graphite-statsd:1.1.8-1 + hostname: graphite + ports: + - 81:80 # nginx-gunicorn-graphite + #- 8080:8080 # gunicorn graphite + - 2003-2004:2003-2004 + - 2023-2024:2023-2024 + - 8125:8125/udp + - 8126:8126 + volumes: + - ./graphite/storage:/opt/graphite/storage + environment: + - COLLECTD=1 + - REDIS_TAGDB=1 + + postgresql: + image: postgres:12 + restart: unless-stopped + environment: + POSTGRES_USER: zabbix + POSTGRES_PASSWORD: zabbix + POSTGRES_DB: zabbixdb + volumes: + - ./postgresql/data:/var/lib/postgresql/data + + zabbix-server: + image: zabbix/zabbix-server-pgsql:ubuntu-5.2-latest + restart: unless-stopped + hostname: zabbix-server-container + environment: + DB_SERVER_HOST: postgresql + POSTGRES_USER: zabbix + POSTGRES_PASSWORD: zabbix + POSTGRES_DB: zabbixdb + ZBX_HISTORYSTORAGETYPES: log,text + ZBX_DEBUGLEVEL: 1 + ZBX_HOUSEKEEPINGFREQUENCY: 1 + ZBX_MAXHOUSEKEEPERDELETE: 5000 + ZBX_TRAPPERIMEOUT: 300 + depends_on: + - postgresql + volumes: + - ./zabbix/alertscripts:/usr/lib/zabbix/alertscripts + - ./zabbix/externalscripts:/usr/lib/zabbix/externalscripts:z + - ./zabbix/modules:/var/lib/zabbix/modules:z + - ./zabbix/enc:/var/lib/zabbix/enc:z + - ./zabbix/ssh_keys:/var/lib/zabbix/ssh_keys:z + - ./zabbix/ssl/certs:/var/lib/zabbix/ssl/certs:z + - ./zabbix/ssl/keys:/var/lib/zabbix/ssl/keys:z + - ./zabbix/ssl/ssl_ca:/var/lib/zabbix/ssl/ssl_ca:z + - ./zabbix/snmptraps:/var/lib/zabbix/snmptraps:z + - ./zabbix/mibs:/var/lib/zabbix/mibs:z + - ./zabbix/export:/var/lib/zabbix/export:z + ports: + - 10051:10051 + + zabbix-web: + image: zabbix/zabbix-web-nginx-pgsql:ubuntu-5.2-latest + restart: unless-stopped + environment: + DB_SERVER_HOST: postgresql + POSTGRES_USER: zabbix + POSTGRES_PASSWORD: zabbix + POSTGRES_DB: zabbixdb + ZBX_SERVER_HOST: zabbix-server # Zabbix related and Php variables + ZBX_POSTMAXSIZE: 64M + ZBX_MAXEXECUTIONTIME: 500 + depends_on: + - postgresql + - zabbix-server + + zabbix-agent: + image: zabbix/zabbix-agent:ubuntu-5.2-latest + # privileged: true # access mode for allowing resource access + hostname: zabbix-agent-container + restart: unless-stopped + environment: + - ZBX_SERVER_HOST=zabbix-server + ports: + - 10050:10050 diff --git a/roles/monitoring_hub/files/monitoring_hub/init-letsencrypt.sh b/roles/monitoring_hub/files/monitoring_hub/init-letsencrypt.sh new file mode 100644 index 0000000..37146de --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/init-letsencrypt.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# based on https://github.com/wmnnd/nginx-certbot +# see https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71 +# +set -xe + + +if ! [ -x "$(command -v docker-compose)" ]; then + echo 'Error: docker-compose is not installed.' >&2 + exit 1 +fi + +domains=(grafana.orgbackbone.org zabbix.orgbackbone.org) +rsa_key_size=4096 +data_path="./nginx/certbot" +email="support@orgbackbone.org" # Adding a valid address is strongly recommended +staging=0 # Set to 1 if you're testing your setup to avoid hitting request limits + +if [ -d "$data_path" ]; then + read -p "Existing data found for $domains. Continue and replace existing certificate? (y/N) " decision + if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then + exit + fi +fi + + +if [ ! -e "$data_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$data_path/conf/ssl-dhparams.pem" ]; then + echo "### Downloading recommended TLS parameters ..." + mkdir -p "$data_path/conf" + curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$data_path/conf/options-ssl-nginx.conf" + curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$data_path/conf/ssl-dhparams.pem" + echo +fi + +echo "### Creating dummy certificate for $domains ..." +path="/etc/letsencrypt/live/$domains" +mkdir -p "$data_path/conf/live/$domains" +docker-compose run --rm --entrypoint "\ + openssl req -x509 -nodes -newkey rsa:$rsa_key_size -days 1\ + -keyout '$path/privkey.pem' \ + -out '$path/fullchain.pem' \ + -subj '/CN=localhost'" certbot +echo + + +echo "### Starting nginx ..." +docker-compose up --force-recreate -d nginx +echo + +echo "### Deleting dummy certificate for $domains ..." +docker-compose run --rm --entrypoint "\ + rm -Rf /etc/letsencrypt/live/$domains && \ + rm -Rf /etc/letsencrypt/archive/$domains && \ + rm -Rf /etc/letsencrypt/renewal/$domains.conf" certbot +echo + + +echo "### Requesting Let's Encrypt certificate for $domains ..." +#Join $domains to -d args +domain_args="" +for domain in "${domains[@]}"; do + domain_args="$domain_args -d $domain" +done + +# Select appropriate email arg +case "$email" in + "") email_arg="--register-unsafely-without-email" ;; + *) email_arg="--email $email" ;; +esac + +# Enable staging mode if needed +if [ $staging != "0" ]; then staging_arg="--staging"; fi + +docker-compose run --rm --entrypoint "\ + certbot certonly --webroot -w /var/www/certbot \ + $staging_arg \ + $email_arg \ + $domain_args \ + --rsa-key-size $rsa_key_size \ + --agree-tos \ + --no-eff-email \ + --force-renewal" certbot +echo + +echo "### Reloading nginx ..." +docker-compose exec nginx nginx -s reload diff --git a/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/certbot-challenge.conf b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/certbot-challenge.conf new file mode 100644 index 0000000..bdf5910 --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/certbot-challenge.conf @@ -0,0 +1,10 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + + location /.well-known/acme-challenge/ { root /var/www/certbot; } + location / { return 301 https://$host$request_uri; } +} + +# add docker's embedded DNS 127.0.0.11 to route docker services with container name +resolver 127.0.0.11 127.0.0.1 [::1]:5353 valid=5s; diff --git a/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/grafana.conf b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/grafana.conf new file mode 100644 index 0000000..7be8425 --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/grafana.conf @@ -0,0 +1,26 @@ +server { + listen 443 ssl; + server_name grafana.orgbackbone.org; + + ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem; + + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + auth_basic ""; + auth_basic_user_file /etc/nginx/nginx-auth.htpasswd; + + location / { + proxy_pass http://grafana:3000; + proxy_set_header Host $host; + proxy_set_header Referer ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_http_version 1.1; + } +} + + diff --git a/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/zabbix.conf b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/zabbix.conf new file mode 100644 index 0000000..d19e651 --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/nginx/conf.d/zabbix.conf @@ -0,0 +1,26 @@ +server { + listen 443 ssl; + server_name zabbix.orgbackbone.org; + + ssl_certificate /etc/letsencrypt/live/grafana.orgbackbone.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.orgbackbone.org/privkey.pem; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + auth_basic ""; + auth_basic_user_file /etc/nginx/nginx-auth.htpasswd; + + location / { + proxy_pass http://zabbix-web:8080/; + proxy_set_header Host $host; + proxy_set_header Referer ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_http_version 1.1; + } +} + + diff --git a/roles/monitoring_hub/files/monitoring_hub/nginx/nginx-auth.htpasswd b/roles/monitoring_hub/files/monitoring_hub/nginx/nginx-auth.htpasswd new file mode 100644 index 0000000..52afc08 --- /dev/null +++ b/roles/monitoring_hub/files/monitoring_hub/nginx/nginx-auth.htpasswd @@ -0,0 +1,3 @@ +USER:um2j1VlJgPYvw + + diff --git a/roles/monitoring_hub/tasks/main.yml b/roles/monitoring_hub/tasks/main.yml index d80ef8d..e61fada 100644 --- a/roles/monitoring_hub/tasks/main.yml +++ b/roles/monitoring_hub/tasks/main.yml @@ -1,3 +1,4 @@ +# just dockprom installer - name: deploy dockprom copy: src: files/dockprom @@ -15,3 +16,28 @@ args: chdir: /opt/dockprom warn: false + +# monitoring_hub: graphite + grafana + zabbix +- name: copy monitoring_hub configs + copy: + src: files/monitoring_hub + dest: /opt + owner: dev + +- name: generate nginx auth + shell: | + printf "USER:$(openssl passwd -crypt PASSWORD)\n" >> nginx/nginx-auth.htpasswd + args: + chdir: /opt/monitoring_hub + +- name: init-letsencrypt.sh + shell: | + bash init-letsencrypt.sh + args: + chdir: /opt/monitoring_hub + +- name: build and start docker containers + shell: | + docker-compose up -d + args: + chdir: /opt/monitoring_hub