PMM-13391 Integrate VictoriaLogs

build/ansible/pmm-docker/victorialogs.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  become: yes
+  gather_facts: yes
+  vars:
+    victorialogs_version: "0.42.0"
+  roles:
+    - victorialogs

build/ansible/roles/victorialogs/files/Dockerfile.victorialogs

@@ -0,0 +1,18 @@
+# TODO: the image should be percona/pmm-server:3 once PMM v3 is released.
+
+# To build the image, run the following in the project root directory (mind the dot!):
+# docker buildx build --platform=linux/amd64 --progress=plain -t perconalab/pmm-server:victorialogs-0.42.0 -f ./build/ansible/roles/victorialogs/files/Dockerfile.victorialogs .
+FROM perconalab/pmm-server:3-dev-latest
+
+ENV GF_ANALYTICS_CHECK_FOR_UPDATES=false
+ENV GF_ANALYTICS_REPORTING_ENABLED=false
+
+USER root
+
+COPY build/ansible /opt/ansible
+
+RUN ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm-docker/victorialogs.yml
+
+EXPOSE 9428
+
+USER pmm

build/ansible/roles/victorialogs/files/json_log_format.conf

@@ -0,0 +1,12 @@
+        'time="$time_iso8601" '
+        'host=$http_host '
+        'remote_addr=$remote_addr '
+        'request_method=$request_method '
+        'request="$request_uri" '
+        'request_time=$request_time '
+        'body_bytes_sent=$body_bytes_sent '
+        'server_protocol="$server_protocol" '
+        'status=$status '
+        'http_referrer="$http_referer" '
+        'http_x_forwarded_for="$http_x_forwarded_for" '
+        'http_user_agent="$http_user_agent"';

build/ansible/roles/victorialogs/files/promtail.yml

@@ -0,0 +1,86 @@
+# Important: too much scraping during init process can overload the system.
+# https://github.com/grafana/loki/issues/11398
+
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+- url: http://127.0.0.1:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host&_msg_field=msg&_time_field=time&ignore_fields=filename,stream
+
+scrape_configs:
+- job_name: nginx
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: nginx
+      __path__: /srv/logs/nginx.log
+
+- job_name: grafana
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: grafana
+      __path__: /srv/logs/grafana.log
+
+- job_name: pmm-agent
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: pmm-agent
+      __path__: /srv/logs/pmm-agent.log
+      node_name: pmm-server
+
+- job_name: pmm-managed
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: pmm-managed
+      __path__: /srv/logs/pmm-managed.log
+
+- job_name: qan
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: qan
+      __path__: /srv/logs/qani-api2.log
+
+- job_name: victoriametrics
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: victoriametrics
+      __path__: /srv/logs/victoriametrics.log
+
+- job_name: clickhouse
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: clickhouse
+      __path__: /srv/logs/clickhouse-server.log
+
+- job_name: supervisor
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: supervisor
+      __path__: /srv/logs/supervisord.log
+
+- job_name: promtail
+  static_configs:
+  - targets:
+      - 127.0.0.1
+    labels:
+      job: promtail
+      __path__: /srv/logs/promtail.log

build/ansible/roles/victorialogs/files/victorialogs.ini

@@ -0,0 +1,35 @@
+[program:victoria-logs]
+priority = 20
+command = /usr/local/bin/victoria-logs 
+    -storageDataPath=/srv/victorialogs/data
+    -retentionPeriod=30d
+    -httpListenAddr=0.0.0.0:9428
+    -envflag.enable
+    -envflag.prefix=VL_
+user = pmm
+autorestart = true
+autostart = true
+startretries = 1000
+startsecs = 3
+stopsignal = TERM
+stopwaitsecs = 10
+stdout_logfile = /srv/logs/victorialogs.log
+stdout_logfile_maxbytes = 20MB
+stdout_logfile_backups = 3
+redirect_stderr = true
+
+
+[program:promtail]
+priority = 21
+command = /usr/bin/promtail -config.file /srv/victorialogs/promtail.yml
+user = pmm
+autorestart = true
+autostart = true
+startretries = 1000
+startsecs = 3
+stopsignal = TERM
+stopwaitsecs = 10
+stdout_logfile = /srv/logs/promtail.log
+stdout_logfile_maxbytes = 20MB
+stdout_logfile_backups = 3
+redirect_stderr = true

build/ansible/roles/victorialogs/files/victorialogs.yml

@@ -0,0 +1,16 @@
+# This is a Grafana datasource to interact with a Loki server
+
+apiVersion: 1
+datasources:
+    # <string, required> Name of the VictoriaLogs datasource
+    # displayed in Grafana panels and queries.
+  - name: VictoriaLogs
+    # <string, required> Sets the data source type.
+    type: victorialogs-datasource
+    uid: victorialogs
+    # <string, required> Sets the access mode, either
+    # proxy or direct (Server or Browser in the UI).
+    access: proxy
+    # <string> Sets URL for sending queries to VictoriaLogs server.
+    # see https://docs.victoriametrics.com/victorialogs/querying/
+    url: http://127.0.0.1:9428

build/ansible/roles/victorialogs/files/vlogs_location.conf

@@ -0,0 +1,13 @@
+    location /vmui {
+      proxy_pass http://127.0.0.1:9428/select/vmui;
+      proxy_http_version 1.1;
+      proxy_read_timeout 600;
+      proxy_set_header Connection "";
+    }
+
+    location /select/logsql {
+      proxy_pass http://127.0.0.1:9428$request_uri;
+      proxy_http_version 1.1;
+      proxy_read_timeout 600;
+      proxy_set_header Connection "";
+    }

build/ansible/roles/victorialogs/tasks/main.yml

@@ -0,0 +1,144 @@
+---
+# This role installs and configures Victorialogs (https://docs.victoriametrics.com/victorialogs/quickstart/).
+
+- name: Download and unarchive AWS installer
+  unarchive:
+    src: "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v{{ victorialogs_version }}-victorialogs/victoria-logs-linux-amd64-v{{ victorialogs_version }}-victorialogs.tar.gz"
+    dest: /tmp
+    remote_src: yes
+
+- name: Copy victorialogs binary
+  copy:
+    src: /tmp/victoria-logs-prod
+    dest: /usr/local/bin/victoria-logs
+    mode: 0755
+
+- name: Remove the temporary file
+  file:
+    path: /tmp/victoria-logs-prod
+    state: absent
+
+- name: Install unzip
+  yum:
+    name: unzip
+    state: latest
+
+- name: Download and unarchive victorialogs-datasource
+  unarchive:
+    src: "https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.2.1/victorialogs-datasource-v0.2.1.zip"
+    dest: /srv/grafana/plugins
+    remote_src: yes
+
+- name: Remove redundant binaries
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_darwin_amd64
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_darwin_arm64
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_linux_386
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_linux_arm
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_linux_arm64
+    - /srv/grafana/plugins/victorialogs-datasource/victorialogs_backend_plugin_windows_amd64.exe
+
+- name: Change ownership of victorialogs-datasource
+  file:
+    path: /srv/grafana/plugins/victorialogs-datasource
+    owner: pmm
+    group: pmm
+    recurse: yes
+
+- name: Add Grafana RPM repository
+  yum_repository:
+    name: grafana
+    description: Grafana repository
+    baseurl: https://rpm.grafana.com
+    gpgcheck: yes
+    enabled: no
+    gpgkey: https://rpm.grafana.com/gpg.key
+    sslverify: yes
+    sslcacert: /etc/pki/tls/certs/ca-bundle.crt
+
+- name: Install Promtail
+  yum:
+    name: promtail
+    state: latest
+    disablerepo: "*"
+    enablerepo: grafana
+
+- name: Create promtail user
+  user:
+    name: "{{ item.name }}"
+    uid: "{{ item.uid }}"
+    comment: "{{ item.comment }}"
+    shell: "{{ item.shell }}"
+    group: "{{ item.group }}"
+    create_home: false
+  loop:
+    - { name: promtail, uid: 1001, comment: "Promtail user", shell: "/sbin/nologin", group: pmm, }
+
+- name: Create a directory for VictoriaLogs
+  file:
+    path: /srv/victorialogs/data
+    state: directory
+    owner: pmm
+    group: pmm
+    recurse: yes
+
+- name: Copy Supervisord config file for VictoriaLogs and Promtail
+  copy:
+    src: victorialogs.ini
+    dest: /etc/supervisord.d/victorialogs.ini
+    owner: pmm
+    group: pmm
+
+- name: Copy Promtail config file
+  copy:
+    src: promtail.yml
+    dest: /srv/victorialogs/promtail.yml
+    owner: pmm
+    group: pmm
+
+- name: Copy VictoriaLogs datasource file
+  copy:
+    src: victorialogs.yml
+    dest: /usr/share/grafana/conf/provisioning/datasources/victorialogs.yml
+    owner: pmm
+    group: pmm
+
+- name: Add victorialogs-datasource to unsigned plugins
+  lineinfile:
+    dest: /etc/grafana/grafana.ini
+    regexp: '^allow_loading_unsigned_plugins'
+    line: 'allow_loading_unsigned_plugins = grafana-polystat-panel,pmm-app,pmm-check-panel-home,pmm-update,pmm-qan-app-panel,pmm-pt-summary-panel,pmm-pt-summary-datasource,victorialogs-datasource'
+    state: present
+
+- name: Remove the end of log_format line in nginx.conf
+  command: sed -i 's/^\([[:space:]]*log_format  main\).*/\1/' /etc/nginx/nginx.conf
+
+- name: Remove the log_format in nginx.conf
+  command: sed -i '/log_format/{n;N;d;}' /etc/nginx/nginx.conf
+
+- name: Replace the original log_format with a json one
+  blockinfile:
+    path: /etc/nginx/nginx.conf
+    marker: "{mark}"
+    marker_begin: "#LOG_FORMAT_BEGIN"
+    marker_end: "#LOG_FORMAT_END"
+    insertafter: "log_format"
+    block: "{{ lookup('ansible.builtin.file', '/opt/ansible/roles/victorialogs/files/json_log_format.conf') }}"
+
+- name: Remove ansible markers from nginx.conf
+  command: sed -i '/^#LOG_FORMAT_.*/d' /etc/nginx/nginx.conf
+
+- name: Add VictoriaLogs location to pmm.conf
+  blockinfile:
+    path: /etc/nginx/conf.d/pmm.conf
+    marker: "{mark}"
+    marker_begin: "#VLOGS_LOCATION_BEGIN"
+    marker_end: "#VLOGS_LOCATION_END"
+    insertbefore: "# Swagger UI"
+    block: "{{ lookup('ansible.builtin.file', '/opt/ansible/roles/victorialogs/files/vlogs_location.conf') }}"
+
+- name: Remove ansible markers from pmm.conf
+  command: sed -i '/^#VLOGS_LOCATION_.*/d' /etc/nginx/conf.d/pmm.conf