split observability stack into two modes (#2047)

Split the observability stack into two modes

Author: skudasov

book/src/framework/observability/observability_stack.md

@@ -2,14 +2,33 @@
 
 You can use a local observability stack, framework is connected to it by default
 
+Default mode includes:
+- Prometheus
+- Loki
+- OTEL-collector
+- Grafana
 ```bash
+# start the observability stack
 ctf obs up
+# remove the stack with all the data (volumes)
+ctf obs d
+# restart the stack removing all the data (volumes)
+ctf obs r
 ```
 
-To remove it use
+Full stack has all the services above but also adds:
+- Cadvisor
+- Tempo
+- Pyroscope
+- PostgreSQL exporters for NodeSet databases
 
 ```bash
-ctf obs down
+# start the observability stack
+ctf obs up -f
+# remove the stack with all the data (volumes)
+ctf obs d -f
+# restart the stack removing all the data (volumes)
+ctf obs r -f
 ```
 
 Read more about how to check [logs](logs.md) and [profiles](profiling.md)

framework/.changeset/v0.10.14.md

@@ -0,0 +1 @@
+- Split the observability stack into two modes: full and standard

framework/cmd/main.go

@@ -68,28 +68,61 @@ func main() {
 				Usage:   "Spins up a local observability stack: Grafana, Loki, Pyroscope",
 				Subcommands: []*cli.Command{
 					{
-						Name:        "up",
-						Usage:       "ctf obs up",
-						Aliases:     []string{"u"},
-						Description: "Spins up a local observability stack: Grafana, Loki, Pyroscope",
-						Action:      func(c *cli.Context) error { return framework.ObservabilityUp() },
+						Name:    "up",
+						Usage:   "ctf obs up",
+						Aliases: []string{"u"},
+						Flags: []cli.Flag{
+							&cli.BoolFlag{
+								Name:    "full",
+								Aliases: []string{"f"},
+								Usage:   "Spin up all the observability services",
+								Value:   false,
+							},
+						},
+						Description: "Spins up a local observability stack. Has two modes, standard (Loki, Prometheus, Grafana and OTEL) and full including also Tempo, Cadvisor and PostgreSQL metrics",
+						Action: func(c *cli.Context) error {
+							if c.Bool("full") {
+								return framework.ObservabilityUpFull()
+							}
+							return framework.ObservabilityUp()
+						},
 					},
 					{
-						Name:        "down",
-						Usage:       "ctf obs down",
-						Aliases:     []string{"d"},
+						Name:    "down",
+						Usage:   "ctf obs down",
+						Aliases: []string{"d"},
+						Flags: []cli.Flag{
+							&cli.BoolFlag{
+								Name:    "full",
+								Aliases: []string{"f"},
+								Usage:   "Removes all the observability services (this flag exists for compatibility, all the services are always removed with 'down')",
+								Value:   false,
+							},
+						},
 						Description: "Removes local observability stack",
 						Action:      func(c *cli.Context) error { return framework.ObservabilityDown() },
 					},
 					{
-						Name:        "restart",
-						Usage:       "ctf obs r",
-						Aliases:     []string{"r"},
+						Name:    "restart",
+						Usage:   "ctf obs r",
+						Aliases: []string{"r"},
+						Flags: []cli.Flag{
+							&cli.BoolFlag{
+								Name:    "full",
+								Aliases: []string{"f"},
+								Usage:   "Restart all observability services (this flag exists for compatibility, all the services are always removed with 'down')",
+								Value:   false,
+							},
+						},
 						Description: "Restart a local observability stack",
 						Action: func(c *cli.Context) error {
+							// always remove all the containers and volumes to clean up the data
 							if err := framework.ObservabilityDown(); err != nil {
 								return err
 							}
+							if c.Bool("full") {
+								return framework.ObservabilityUpFull()
+							}
 							return framework.ObservabilityUp()
 						},
 					},

framework/observability.go

@@ -124,6 +124,30 @@ func ObservabilityUp() error {
 	if err := NewPromtail(); err != nil {
 		return err
 	}
+	err := RunCommand("bash", "-c", fmt.Sprintf(`
+		cd %s && \
+		docker compose up -d otel-collector prometheus loki grafana
+	`, "compose"))
+	if err != nil {
+		return err
+	}
+	fmt.Println()
+	L.Info().Msgf("Loki: %s", LocalLogsURL)
+	L.Info().Msgf("Prometheus: %s", LocalPrometheusURL)
+	L.Info().Msgf("CL Node Errors: %s", LocalCLNodeErrorsURL)
+	L.Info().Msgf("Workflow Engine: %s", LocalWorkflowEngineURL)
+	return nil
+}
+
+func ObservabilityUpFull() error {
+	L.Info().Msg("Creating full local observability stack")
+	if err := extractAllFiles("observability"); err != nil {
+		return err
+	}
+	_ = DefaultNetwork(nil)
+	if err := NewPromtail(); err != nil {
+		return err
+	}
 	err := RunCommand("bash", "-c", fmt.Sprintf(`
 		cd %s && \
 		docker compose up -d

framework/observability/compose/docker-compose.yaml

@@ -82,7 +82,6 @@ services:
     depends_on:
       - prometheus
       - loki
-      - tempo
 
   pyroscope:
     image: 'grafana/pyroscope:1.13.4'

framework/observability/dashboards/dummy.json

@@ -0,0 +1,3 @@
+{
+  "example": "This file is needed for local testing so embedding can work."
+}