PMM-13057 New telemetry endpoint. (#3230)

* PMM-13057 Fix imports. * WIP on PMM-13057-new-telemetry * PMM-13057 Changes. * PMM-13057 Tidy. * PMM-13057 Tidy. * PMM-13057 Remove print. * PMM-13057 Fix test. * PMM-13057 Refactor. * PMM-13057 Format. * PMM-13057 Fix for UUIDs format. * PMM-13057 Add product family field. * PMM-13057 Requested renaming. * PMM-13057 Fix for changes. --------- Co-authored-by: Nurlan Moldomurov <[email protected]> Co-authored-by: Alex Demidoff <[email protected]>
percona · Nov 8, 2024 · 7834ff9 · 7834ff9
1 parent 11e5142
commit 7834ff9
Show file tree

Hide file tree

Showing 43 changed files with 171 additions and 163 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -21,7 +21,9 @@ linters-settings:
           - pkg: "github.com/percona/go-mysql/log/slow"
             desc: only use the forked parser
           - pkg: "github.com/percona-platform/platform"
-            desc: use "github.com/percona-platform/saas" instead
+            desc: use "github.com/percona/saas" instead
+          - pkg: "github.com/percona/platform"
+            desc: use "github.com/percona/saas" instead
           - pkg: "gopkg.in/yaml.v2"
             desc: use "gopkg.in/yaml.v3" instead
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -18,7 +18,7 @@ This project is built from several repositories:
 ### APIs
 
 * [percona/pmm](https://github.com/percona/pmm/tree/main/api)
-* [percona-platform/saas](https://github.com/percona-platform/saas)
+* [percona/saas](https://github.com/percona/saas)
 
 ### PMM Server
 

diff --git a/api-tests/alerting/alerting_test.go b/api-tests/alerting/alerting_test.go
@@ -31,7 +31,7 @@ import (
 	"github.com/AlekSi/pointer"
 	"github.com/google/uuid"
 	gapi "github.com/grafana/grafana-api-golang-client"
-	"github.com/percona-platform/saas/pkg/alert"
+	"github.com/percona/saas/pkg/alert"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc/codes"

diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/percona/pmm
 go 1.23.2
 
 // Update saas with
-// go get -v github.com/percona-platform/saas@latest
+// go get -v github.com/percona/saas@latest
 
 replace github.com/grpc-ecosystem/go-grpc-prometheus => github.com/Percona-Lab/go-grpc-prometheus v0.0.0-20230116133345-3487748d4592
 
@@ -51,11 +51,11 @@ require (
 	github.com/jotaen/kong-completion v0.0.5
 	github.com/lib/pq v1.10.9
 	github.com/minio/minio-go/v7 v7.0.55
-	github.com/percona-platform/saas v0.0.0-20230728161159-ad6bdeb8a3d9
 	github.com/percona/exporter_shared v0.7.5
 	github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c
 	github.com/percona/percona-toolkit v3.2.1+incompatible
 	github.com/percona/promconfig v0.2.5
+	github.com/percona/saas v0.0.0-20240923141535-da19f6682c6e
 	github.com/pganalyze/pg_query_go/v5 v5.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.6

diff --git a/go.sum b/go.sum
@@ -410,8 +410,6 @@ github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/En
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2 h1:FaO5loTAkRzVhyKWYRHg/ii9oWXNsqv89H/kGl9Cb5I=
 github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2/go.mod h1:aSyBXtGhRzSMdne9jbl3+PPMVS0IgOWF6Llc+HB5uUU=
-github.com/percona-platform/saas v0.0.0-20230728161159-ad6bdeb8a3d9 h1:KkOH+Y4sVRP7qvRtTDmfPFNjjQcwU2054/jNl9DZhEo=
-github.com/percona-platform/saas v0.0.0-20230728161159-ad6bdeb8a3d9/go.mod h1:lZuFcqj0EoQWx28SYkTcdhJOCQEbRcAyahYPfRMY7tc=
 github.com/percona/exporter_shared v0.7.5 h1:sIhdDtKOfH0jsqtTAi41C7wm1R42yXYeAkK0QWhgN7k=
 github.com/percona/exporter_shared v0.7.5/go.mod h1:P8ZHDzveyJRGD4ZIRNiEMuAdDexBpQWf2mAVwtyVcYU=
 github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c h1:1SZ7nS+kSaO63IpaKspf/gf8602QcgP2eXNPMNOIc0M=
@@ -420,6 +418,8 @@ github.com/percona/percona-toolkit v3.2.1+incompatible h1:5jLvtZKcu9fDmaLRB8qA4b
 github.com/percona/percona-toolkit v3.2.1+incompatible/go.mod h1:netQWdWMaF1cnmwiIS+i5uyaqNXz46yNeM6HKkR6yeI=
 github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yigquc=
 github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
+github.com/percona/saas v0.0.0-20240923141535-da19f6682c6e h1:xhk5ivxlTPat0SwLBqU1UdfqJb+2R3x4yAhjCf6WMEU=
+github.com/percona/saas v0.0.0-20240923141535-da19f6682c6e/go.mod h1:UP4WqGWy8xeGhEoDFaB1aPNNtNs/TYXQ8MjUrN6rXis=
 github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg=
 github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=

diff --git a/managed/cmd/pmm-managed-starlark/main.go b/managed/cmd/pmm-managed-starlark/main.go
@@ -24,8 +24,8 @@ import (
 	"time"
 
 	"github.com/alecthomas/kingpin/v2"
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/starlark"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/starlark"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.starlark.net/resolve"

diff --git a/managed/cmd/pmm-managed-starlark/main_test.go b/managed/cmd/pmm-managed-starlark/main_test.go
@@ -24,7 +24,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/percona-platform/saas/pkg/check"
+	"github.com/percona/saas/pkg/check"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

diff --git a/managed/models/template_helpers.go b/managed/models/template_helpers.go
@@ -19,7 +19,7 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
-	"github.com/percona-platform/saas/pkg/alert"
+	"github.com/percona/saas/pkg/alert"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"

diff --git a/managed/models/template_helpers_test.go b/managed/models/template_helpers_test.go
@@ -22,9 +22,9 @@ import (
 	"github.com/AlekSi/pointer"
 	"github.com/brianvoe/gofakeit/v6"
 	"github.com/google/uuid"
-	"github.com/percona-platform/saas/pkg/alert"
-	"github.com/percona-platform/saas/pkg/common"
 	"github.com/percona/promconfig"
+	"github.com/percona/saas/pkg/alert"
+	"github.com/percona/saas/pkg/common"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/reform.v1"

diff --git a/managed/models/template_model.go b/managed/models/template_model.go
@@ -19,7 +19,7 @@ import (
 	"database/sql/driver"
 	"time"
 
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/common"
 	"github.com/pkg/errors"
 	"gopkg.in/reform.v1"
 )

diff --git a/managed/services/alerting/service.go b/managed/services/alerting/service.go
@@ -30,8 +30,8 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
-	"github.com/percona-platform/saas/pkg/alert"
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/alert"
+	"github.com/percona/saas/pkg/common"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"

diff --git a/managed/services/checks/checks.go b/managed/services/checks/checks.go
@@ -32,7 +32,7 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/percona-platform/saas/pkg/check"
+	"github.com/percona/saas/pkg/check"
 	"github.com/pkg/errors"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	prom "github.com/prometheus/client_golang/prometheus"

diff --git a/managed/services/checks/checks_test.go b/managed/services/checks/checks_test.go
@@ -24,8 +24,8 @@ import (
 
 	"github.com/AlekSi/pointer"
 	_ "github.com/ClickHouse/clickhouse-go/v2"
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/common"
 	metrics "github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/sirupsen/logrus"

diff --git a/managed/services/checks/funcs.go b/managed/services/checks/funcs.go
@@ -19,7 +19,7 @@ import (
 	"net"
 	"strconv"
 
-	"github.com/percona-platform/saas/pkg/starlark"
+	"github.com/percona/saas/pkg/starlark"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 

diff --git a/managed/services/checks/funcs_test.go b/managed/services/checks/funcs_test.go
@@ -19,9 +19,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/common"
-	"github.com/percona-platform/saas/pkg/starlark"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/common"
+	"github.com/percona/saas/pkg/starlark"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

diff --git a/managed/services/checks/registry.go b/managed/services/checks/registry.go
@@ -18,7 +18,7 @@ package checks
 import (
 	"sync"
 
-	"github.com/percona-platform/saas/pkg/check"
+	"github.com/percona/saas/pkg/check"
 	prom "github.com/prometheus/client_golang/prometheus"
 
 	"github.com/percona/pmm/managed/services"

diff --git a/managed/services/checks/registry_test.go b/managed/services/checks/registry_test.go
@@ -18,8 +18,8 @@ package checks
 import (
 	"testing"
 
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/common"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

diff --git a/managed/services/management/checks.go b/managed/services/management/checks.go
@@ -20,8 +20,8 @@ import (
 	"strings"
 
 	"github.com/AlekSi/pointer"
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/common"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"

diff --git a/managed/services/management/checks_test.go b/managed/services/management/checks_test.go
@@ -21,8 +21,8 @@ import (
 	"testing"
 
 	"github.com/AlekSi/pointer"
-	"github.com/percona-platform/saas/pkg/check"
-	"github.com/percona-platform/saas/pkg/common"
+	"github.com/percona/saas/pkg/check"
+	"github.com/percona/saas/pkg/common"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"

diff --git a/managed/services/management/deps.go b/managed/services/management/deps.go
@@ -19,7 +19,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/percona-platform/saas/pkg/check"
+	"github.com/percona/saas/pkg/check"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
 	"gopkg.in/reform.v1"

diff --git a/managed/services/management/mock_checks_service_test.go b/managed/services/management/mock_checks_service_test.go
diff --git a/managed/services/telemetry/datasource_envvars.go b/managed/services/telemetry/datasource_envvars.go
@@ -20,7 +20,7 @@ import (
 	"context"
 	"os"
 
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/sirupsen/logrus"
 )
 
@@ -51,8 +51,8 @@ func (d *dsEnvvars) Init(_ context.Context) error {
 	return nil
 }
 
-func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
-	var metrics []*pmmv1.ServerMetric_Metric
+func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*telemetryv1.GenericReport_Metric, error) {
+	var metrics []*telemetryv1.GenericReport_Metric
 
 	check := make(map[string]bool, len(config.Data))
 
@@ -69,7 +69,7 @@ func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.Ser
 
 			check[col.MetricName] = true
 
-			metrics = append(metrics, &pmmv1.ServerMetric_Metric{
+			metrics = append(metrics, &telemetryv1.GenericReport_Metric{
 				Key:   col.MetricName,
 				Value: value,
 			})

diff --git a/managed/services/telemetry/datasource_envvars_test.go b/managed/services/telemetry/datasource_envvars_test.go
@@ -21,7 +21,7 @@ import (
 	"os"
 	"testing"
 
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -94,7 +94,7 @@ func TestEnvVarsDatasource(t *testing.T) {
 		metrics, err := dsEnvVars.FetchMetrics(ctx, *config)
 		require.NoError(t, err)
 
-		expected := []*pmmv1.ServerMetric_Metric{
+		expected := []*telemetryv1.GenericReport_Metric{
 			{Key: "test_env_var1", Value: "1"},
 			{Key: "test_env_var2", Value: "test"},
 			{Key: "test_env_var3", Value: "true"},
@@ -138,7 +138,7 @@ func TestEnvVarsDatasource(t *testing.T) {
 		metrics, err := dsEnvVars.FetchMetrics(ctx, *config)
 		require.NoError(t, err)
 
-		expected := []*pmmv1.ServerMetric_Metric{
+		expected := []*telemetryv1.GenericReport_Metric{
 			{Key: "test_env_var6", Value: "1"},
 			{Key: "test_env_var7", Value: "1"},
 			{Key: "test_env_var8", Value: "1"},

diff --git a/managed/services/telemetry/datasource_grafanadb_select.go b/managed/services/telemetry/datasource_grafanadb_select.go
@@ -22,8 +22,7 @@ import (
 	"net/url"
 	"time"
 
-	// Events, errors and driver for grafana database.
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -95,7 +94,7 @@ func openGrafanaDBConnection(config DSConfigGrafanaDB, l *logrus.Entry) (*sql.DB
 	return db, nil
 }
 
-func (d *dsGrafanaDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+func (d *dsGrafanaDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*telemetryv1.GenericReport_Metric, error) {
 	return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config)
 }
 

diff --git a/managed/services/telemetry/datasource_pmmdb_select.go b/managed/services/telemetry/datasource_pmmdb_select.go
@@ -22,7 +22,7 @@ import (
 	"net/url"
 	"time"
 
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -94,7 +94,7 @@ func openPMMDBConnection(config DSConfigPMMDB, l *logrus.Entry) (*sql.DB, error)
 	return db, nil
 }
 
-func (d *dsPmmDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+func (d *dsPmmDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*telemetryv1.GenericReport_Metric, error) {
 	return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config)
 }
 

diff --git a/managed/services/telemetry/datasource_qandb_select.go b/managed/services/telemetry/datasource_qandb_select.go
@@ -20,7 +20,7 @@ import (
 	"context"
 	"database/sql"
 
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -69,7 +69,7 @@ func openQANDBConnection(dsn string, enabled bool, l *logrus.Entry) (*sql.DB, er
 	return db, nil
 }
 
-func (d *dsQanDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+func (d *dsQanDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*telemetryv1.GenericReport_Metric, error) {
 	return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config)
 }
 

diff --git a/managed/services/telemetry/datasource_victoria_metrics.go b/managed/services/telemetry/datasource_victoria_metrics.go
@@ -20,7 +20,7 @@ import (
 	"context"
 	"time"
 
-	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	telemetryv1 "github.com/percona/saas/gen/telemetry/generic"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
@@ -66,7 +66,7 @@ func NewDataSourceVictoriaMetrics(config DSConfigVM, l *logrus.Entry) (DataSourc
 	}, nil
 }
 
-func (d *dataSourceVictoriaMetrics) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+func (d *dataSourceVictoriaMetrics) FetchMetrics(ctx context.Context, config Config) ([]*telemetryv1.GenericReport_Metric, error) {
 	localCtx, cancel := context.WithTimeout(ctx, d.config.Timeout)
 	defer cancel()
 
@@ -75,20 +75,20 @@ func (d *dataSourceVictoriaMetrics) FetchMetrics(ctx context.Context, config Con
 		return nil, err
 	}
 
-	var metrics []*pmmv1.ServerMetric_Metric
+	var metrics []*telemetryv1.GenericReport_Metric
 
 	for _, v := range result.(model.Vector) { //nolint:forcetypeassert
 		for _, configItem := range config.Data {
 			if configItem.Label != "" {
 				value := v.Metric[model.LabelName(configItem.Label)]
-				metrics = append(metrics, &pmmv1.ServerMetric_Metric{
+				metrics = append(metrics, &telemetryv1.GenericReport_Metric{
 					Key:   configItem.MetricName,
 					Value: string(value),
 				})
 			}
 
 			if configItem.Value != "" {
-				metrics = append(metrics, &pmmv1.ServerMetric_Metric{
+				metrics = append(metrics, &telemetryv1.GenericReport_Metric{
 					Key:   configItem.MetricName,
 					Value: v.Value.String(),
 				})