[Only for testing] Multi cluster ceph dev

.env.example

@@ -22,7 +22,9 @@ PROMETHEUS_ALERTS_DIR=monitoring/ceph-mixin
 #PROMETHEUS_ALERTS_DIR=monitoring/prometheus/alerts
 
 PROMETHEUS_HOST_PORT=9090
+PROMETHEUS_HOST_PORT1=9098
 #PROMETHEUS_IMAGE=
+THANOS_IMAGE=thanosio/thanos:main-2023-10-16-77fac93
 NODE_EXPORTER_HOST_PORT=9100
 #NODE_EXPORTER_IMAGE=
 ALERTMANAGER_HOST_PORT=9093

docker-compose.yml

@@ -121,6 +121,63 @@ services:
             - ./docker/prometheus:/etc/prometheus:Z
             - ${CEPH_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z
 
+    prometheus2:
+        image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.43.0}
+        container_name: prometheus2
+        hostname: prometheus2
+        ports: ['${PROMETHEUS2_HOST_PORT:-9090}:9090']
+        volumes:
+            - ./docker/prometheus1:/etc/prometheus:Z
+            - ${CEPH2_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z
+
+    thanos-sidecar:
+        image: ${THANOS_IMAGE}
+        container_name: thanos-sidecar
+        hostname: thanos-sidecar
+        volumes_from:
+          - prometheus:rw
+        command:
+          - "sidecar"
+          - "--tsdb.path=/prometheus"
+          - "--prometheus.url=http://prometheus:9090"
+        expose:
+          - 10901
+          - 10902
+        depends_on:
+          - prometheus
+
+    thanos-sidecar2:
+        image: ${THANOS_IMAGE}
+        container_name: thanos-sidecar2
+        hostname: thanos-sidecar2
+        volumes_from:
+          - prometheus2:rw
+        command:
+          - "sidecar"
+          - "--tsdb.path=/prometheus"
+          - "--prometheus.url=http://prometheus2:9090"
+        expose:
+          - 10901
+          - 10902
+        depends_on:
+          - prometheus2
+
+    thanos-querier:
+        image: ${THANOS_IMAGE}
+        container_name: thanos-querier
+        hostname: thanos-querier
+        command:
+          - "query"
+          - "--store=thanos-sidecar:10901"
+          - "--store=thanos-sidecar2:10901"
+        expose:
+          - 10902
+          - 10901
+        ports:
+          - "10902:10902"
+        depends_on:
+          - thanos-sidecar
+
     node-exporter:
         image: ${NODE_EXPORTER_IMAGE:-prom/node-exporter:v1.5.0}
         container_name: node-exporter

docker/grafana/provisioning/datasources/prometheus.yml

@@ -11,3 +11,10 @@ datasources:
     orgId: 1
     url: http://prometheus:9090
     editable: true
+
+  - name: Thanos
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://thanos-querier:10902
+    editable: true

docker/prometheus/ceph-targets.yml

@@ -1,10 +1,8 @@
 [
     {
         "targets": [ "ceph:9283" ],
-        "labels": {}
+        "labels": {
+          cluster: "cluster1"
+        }
     },
-    {
-        "targets": [ "ceph2:9283" ],
-        "labels": {}
-    }
-]
+]

docker/prometheus/prometheus.yml

@@ -1,7 +1,8 @@
 global:
   scrape_interval: 5s
   evaluation_interval: 5s
-
+  external_labels:
+    cluster: cluster1
 scrape_configs:
   - job_name: 'node-exporter'
     file_sd_configs:

docker/prometheus1/ceph-targets.yml

@@ -0,0 +1,8 @@
+[
+    {
+        "targets": [ "ceph2:9283" ],
+        "labels": {
+          cluster: "cluster2"
+        }
+    }
+]

docker/prometheus1/node-exporter-targets.yml

@@ -0,0 +1,8 @@
+[
+    {
+        "targets": [ "node-exporter:9100" ],
+        "labels": {
+            "instance": "node-exporter"
+        }
+    }
+]

docker/prometheus1/prometheus.yml

@@ -0,0 +1,26 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+  external_labels:  
+    cluster: cluster2
+scrape_configs:
+  - job_name: 'node-exporter'
+    file_sd_configs:
+      - files:
+        - node-exporter-targets.yml
+  - job_name: 'ceph'
+    honor_labels: true
+    file_sd_configs:
+      - files:
+        - ceph-targets.yml
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets:
+      - 'alertmanager:9094'
+
+rule_files:
+  - 'alerts/ceph_default_alerts.yml'
+  - 'alerts/prometheus_alerts.yml'