access secrets in GCP concurrently

Author: alexei-led

Diff for: Dockerfile

@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:experimental
 
-FROM --platform=${BUILDPLATFORM} golang:1.21-alpine as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine as builder
 # passed by buildkit
 ARG TARGETOS
 ARG TARGETARCH

Diff for: Makefile

@@ -52,7 +52,7 @@ platfrom-build: clean lint test ; $(info $(M) building binaries for multiple os/
 setup-tools: setup-lint setup-gocov setup-gocov-xml setup-go2xunit setup-mockery setup-ghr
 
 setup-lint:
-	$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2
+	$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.1
 setup-gocov:
 	$(GO) install github.com/axw/gocov/...
 setup-gocov-xml:

Diff for: go.mod

@@ -1,6 +1,6 @@
 module secrets-init
 
-go 1.21
+go 1.22
 
 require (
 	cloud.google.com/go/compute v1.10.0

Diff for: go.sum

@@ -1,5 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8=
+cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA=
 cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4=
 cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU=
 cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg=

Diff for: main.go

@@ -80,7 +80,7 @@ func main() {
 		Action:  mainCmd,
 		Version: Version,
 	}
-	cli.VersionPrinter = func(c *cli.Context) {
+	cli.VersionPrinter = func(_ *cli.Context) {
 		fmt.Printf("version: %s\n", Version)
 		fmt.Printf("  build date: %s\n", BuildDate)
 		fmt.Printf("  commit: %s\n", GitCommit)

Diff for: pkg/secrets/google/secrets.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+	"sync"
 
 	"secrets-init/pkg/secrets" //nolint:gci
 
@@ -15,6 +16,13 @@ import (
 	secretspb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" //nolint:gci
 )
 
+var fullSecretRe = regexp.MustCompile(`projects/[^/]+/secrets/[^/+](/version/[^/+])?`)
+
+type result struct {
+	Env string
+	Err error
+}
+
 // SecretsProvider Google Cloud secrets provider
 type SecretsProvider struct {
 	sm        SecretsManagerAPI
@@ -53,40 +61,79 @@ func NewGoogleSecretsProvider(ctx context.Context, projectID string) (secrets.Pr
 func (sp SecretsProvider) ResolveSecrets(ctx context.Context, vars []string) ([]string, error) {
 	envs := make([]string, 0, len(vars))
 
-	fullSecretRe := regexp.MustCompile("projects/[^/]+/secrets/[^/+](/version/[^/+])?")
+	// Create a channel to collect the results
+	results := make(chan result, len(vars))
 
+	// Start a goroutine for each secret
+	var wg sync.WaitGroup
 	for _, env := range vars {
-		kv := strings.Split(env, "=")
-		key, value := kv[0], kv[1]
-		if strings.HasPrefix(value, "gcp:secretmanager:") {
-			// construct valid secret name
-			name := strings.TrimPrefix(value, "gcp:secretmanager:")
-
-			isLong := fullSecretRe.MatchString(name)
-
-			if !isLong {
-				if sp.projectID == "" {
-					return vars, errors.Errorf("failed to get secret \"%s\" from Google Secret Manager (unknown project)", name)
+		wg.Add(1)
+		go func(env string) {
+			defer wg.Done()
+			select {
+			case <-ctx.Done():
+				results <- result{Err: ctx.Err()}
+				return
+			default:
+				val, err := sp.processEnvironmentVariable(ctx, env)
+				if err != nil {
+					results <- result{Err: err}
+					return
 				}
-				name = fmt.Sprintf("projects/%s/secrets/%s", sp.projectID, name)
+				results <- result{Env: val}
 			}
+		}(env)
+	}
 
-			// if no version specified add latest
-			if !strings.Contains(name, "/versions/") {
-				name += "/versions/latest"
-			}
-			// get secret value
-			req := &secretspb.AccessSecretVersionRequest{
-				Name: name,
-			}
-			secret, err := sp.sm.AccessSecretVersion(ctx, req)
-			if err != nil {
-				return vars, errors.Wrap(err, "failed to get secret from Google Secret Manager")
-			}
-			env = key + "=" + string(secret.Payload.GetData())
+	// Start another goroutine to close the results channel when all fetch goroutines are done
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	// Collect the results
+	for res := range results {
+		if res.Err != nil {
+			return vars, res.Err
 		}
-		envs = append(envs, env)
+		envs = append(envs, res.Env)
 	}
 
 	return envs, nil
 }
+
+// processEnvironmentVariable processes the environment variable and replaces the value with the secret value
+func (sp SecretsProvider) processEnvironmentVariable(ctx context.Context, env string) (string, error) {
+	kv := strings.Split(env, "=")
+	key, value := kv[0], kv[1]
+	if !strings.HasPrefix(value, "gcp:secretmanager:") {
+		return env, nil
+	}
+
+	// construct valid secret name
+	name := strings.TrimPrefix(value, "gcp:secretmanager:")
+
+	isLong := fullSecretRe.MatchString(name)
+
+	if !isLong {
+		if sp.projectID == "" {
+			return "", errors.Errorf("failed to get secret \"%s\" from Google Secret Manager (unknown project)", name)
+		}
+		name = fmt.Sprintf("projects/%s/secrets/%s", sp.projectID, name)
+	}
+
+	// if no version specified add latest
+	if !strings.Contains(name, "/versions/") {
+		name += "/versions/latest"
+	}
+
+	// get secret value
+	req := &secretspb.AccessSecretVersionRequest{
+		Name: name,
+	}
+	secret, err := sp.sm.AccessSecretVersion(ctx, req)
+	if err != nil {
+		return "", fmt.Errorf("failed to get secret from Google Secret Manager: %w", err)
+	}
+	return key + "=" + string(secret.Payload.GetData()), nil
+}

Diff for: pkg/secrets/google/secrets_test.go

@@ -4,12 +4,12 @@ package google
 import (
 	"context"
 	"errors"
-	"reflect"
 	"testing"
 
 	"secrets-init/mocks"
 	"secrets-init/pkg/secrets"
 
+	"github.com/stretchr/testify/assert"
 	secretspb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
 
@@ -205,7 +205,7 @@ func TestSecretsProvider_ResolveSecrets(t *testing.T) {
 				t.Errorf("SecretsProvider.ResolveSecrets() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
+			if !assert.ElementsMatch(t, got, tt.want) {
 				t.Errorf("SecretsProvider.ResolveSecrets() = %v, want %v", got, tt.want)
 			}
 			mockSM.AssertExpectations(t)