Basic Secret Source Implementation

Add extensible Secret Sources that can be used to get secrets that will
be redacted from logs.

The in core secret sources are a file and mock based one. One is reading
from key=value file, the other takes secrets as part of its argument.

This likely will be extended in the future with more secure ways to do
secrets.

All secret values are being redacted from the logs of k6 before they go
anywhere. Removing them from other places is not in scope.

Closes #4139

Author: mstoykov

cmd/state/state.go

@@ -14,7 +14,9 @@ import (
 
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
+	"go.k6.io/k6/secretsource"
 )
 
 const defaultConfigFileName = "config.json"
@@ -55,6 +57,9 @@ type GlobalState struct {
 
 	Logger         *logrus.Logger //nolint:forbidigo //TODO:change to FieldLogger
 	FallbackLogger logrus.FieldLogger
+
+	SecretsManager *secretsource.SecretsManager
+	Usage          *usage.Usage
 }
 
 // NewGlobalState returns a new GlobalState with the given ctx.
@@ -131,6 +136,7 @@ func NewGlobalState(ctx context.Context) *GlobalState {
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.InfoLevel,
 		},
+		Usage: usage.New(),
 	}
 }
 
@@ -142,6 +148,7 @@ type GlobalFlags struct {
 	Address          string
 	ProfilingEnabled bool
 	LogOutput        string
+	SecretSource     []string
 	LogFormat        string
 	Verbose          bool
 }

ext/ext.go

@@ -25,6 +25,7 @@ type ExtensionType uint8
 const (
 	JSExtension ExtensionType = iota + 1
 	OutputExtension
+	SecretSourceExtension
 )
 
 func (e ExtensionType) String() string {
@@ -34,6 +35,8 @@ func (e ExtensionType) String() string {
 		s = "js"
 	case OutputExtension:
 		s = "output"
+	case SecretSourceExtension:
+		s = "secret-source"
 	}
 	return s
 }
@@ -157,4 +160,5 @@ func extractModuleInfo(mod interface{}) (path, version string) {
 func init() {
 	extensions[JSExtension] = make(map[string]*Extension)
 	extensions[OutputExtension] = make(map[string]*Extension)
+	extensions[SecretSourceExtension] = make(map[string]*Extension)
 }

internal/cmd/root.go

@@ -19,7 +19,11 @@ import (
 	"go.k6.io/k6/cmd/state"
 	"go.k6.io/k6/errext"
 	"go.k6.io/k6/errext/exitcodes"
+	"go.k6.io/k6/ext"
 	"go.k6.io/k6/internal/log"
+	"go.k6.io/k6/secretsource"
+
+	_ "go.k6.io/k6/internal/secretsource" // import it to register internal secret sources
 )
 
 const waitLoggerCloseTimeout = time.Second * 5
@@ -162,6 +166,10 @@ func rootCmdPersistentFlagSet(gs *state.GlobalState) *pflag.FlagSet {
 	// `gs.DefaultFlags.<value>`, so that the `k6 --help` message is
 	// not messed up...
 
+	// TODO(@mstoykov): likely needs work - no env variables and such. No config.json.
+	flags.StringArrayVar(&gs.Flags.SecretSource, "secret-source", gs.Flags.SecretSource,
+		"setting secret sources for k6 file[=./path.fileformat],")
+
 	flags.StringVar(&gs.Flags.LogOutput, "log-output", gs.Flags.LogOutput,
 		"change the output for k6 logs, possible values are stderr,stdout,none,loki[=host:port],file[=./path.fileformat]")
 	flags.Lookup("log-output").DefValue = gs.DefaultFlags.LogOutput
@@ -257,6 +265,22 @@ func (c *rootCommand) setupLoggers(stop <-chan struct{}) error {
 		c.globalState.Logger.Debug("Logger format: TEXT")
 	}
 
+	secretsources, err := createSecretSources(c.globalState)
+	if err != nil {
+		return err
+	}
+	// it is important that we add this hook first as hooks are executed in order of addition
+	// and this means no other hook will get secrets
+	var secretsHook logrus.Hook
+	c.globalState.SecretsManager, secretsHook, err = secretsource.NewSecretsManager(secretsources)
+	if err != nil {
+		return err
+	}
+	if len(secretsources) != 0 {
+		// don't actually filter anything if there will be no secrets
+		c.globalState.Logger.AddHook(secretsHook)
+	}
+
 	cancel := func() {} // noop as default
 	if hook != nil {
 		ctx := context.Background()
@@ -289,3 +313,49 @@ func (c *rootCommand) setLoggerHook(ctx context.Context, h log.AsyncHook) {
 	c.globalState.Logger.AddHook(h)
 	c.globalState.Logger.SetOutput(io.Discard) // don't output to anywhere else
 }
+
+func createSecretSources(gs *state.GlobalState) (map[string]secretsource.SecretSource, error) {
+	baseParams := secretsource.Params{
+		Logger:      gs.Logger,
+		Environment: gs.Env,
+		FS:          gs.FS,
+		Usage:       gs.Usage,
+	}
+
+	result := make(map[string]secretsource.SecretSource)
+	for _, line := range gs.Flags.SecretSource {
+		t, config, ok := strings.Cut(line, "=")
+		if !ok {
+			return nil, fmt.Errorf("couldn't parse secret source configuration %q", line)
+		}
+		secretSources := ext.Get(ext.SecretSourceExtension)
+		found, ok := secretSources[t]
+		if !ok {
+			return nil, fmt.Errorf("no secret source for type %q for configuration %q", t, line)
+		}
+		c := found.Module.(secretsource.Constructor) //nolint:forcetypeassert
+		params := baseParams
+		params.ConfigArgument = config
+		// TODO(@mstoykov): make it not configurable just from cmd line
+		// params.JSONConfig = test.derivedConfig.Collectors[outputType]
+
+		secretSource, err := c(params)
+		if err != nil {
+			return nil, err
+		}
+		name := secretSource.Name()
+		_, alreadRegistered := result[name]
+		if alreadRegistered {
+			return nil, fmt.Errorf("secret source for name %q already registered before configuration %q", t, line)
+		}
+		result[name] = secretSource
+	}
+
+	if len(result) == 1 {
+		for _, l := range result {
+			result["default"] = l
+		}
+	}
+
+	return result, nil
+}

internal/cmd/test_load.go

@@ -18,7 +18,6 @@ import (
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/internal/js"
 	"go.k6.io/k6/internal/loader"
-	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/js/modules"
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/fsext"
@@ -84,7 +83,8 @@ func loadLocalTest(gs *state.GlobalState, cmd *cobra.Command, args []string) (*l
 			val, ok := gs.Env[key]
 			return val, ok
 		},
-		Usage: usage.New(),
+		Usage:          gs.Usage,
+		SecretsManager: gs.SecretsManager,
 	}
 
 	test := &loadedTest{

internal/cmd/tests/test_state.go

@@ -18,6 +18,7 @@ import (
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/lib/testutils"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
 )
 
@@ -111,6 +112,7 @@ func NewGlobalTestState(tb testing.TB) *GlobalTestState {
 		SignalStop:     signal.Stop,
 		Logger:         logger,
 		FallbackLogger: testutils.NewLogger(tb).WithField("fallback", true),
+		Usage:          usage.New(),
 	}
 
 	return ts

internal/secretsource/file/file.go

@@ -0,0 +1,82 @@
+// Package file implements secret source that reads the secrets from a file as key=value pairs one per line
+package file
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.k6.io/k6/secretsource"
+)
+
+func init() {
+	secretsource.RegisterExtension("file", func(params secretsource.Params) (secretsource.SecretSource, error) {
+		fss := &fileSecretSource{}
+		err := fss.parseArg(params.ConfigArgument)
+		if err != nil {
+			return nil, err
+		}
+
+		f, err := params.FS.Open(fss.filename)
+		if err != nil {
+			return nil, err
+		}
+		scanner := bufio.NewScanner(f)
+
+		fss.internal = make(map[string]string)
+		for scanner.Scan() {
+			line := scanner.Text()
+			k, v, ok := strings.Cut(line, "=")
+			if !ok {
+				return nil, fmt.Errorf("parsing %q, needs =", line)
+			}
+
+			fss.internal[k] = v
+		}
+		return fss, nil
+	})
+}
+
+func (fss *fileSecretSource) parseArg(config string) error {
+	list := strings.Split(config, ",")
+	if len(list) >= 1 {
+		for _, kv := range list {
+			k, v, ok := strings.Cut(kv, "=")
+			if !ok {
+				fss.filename = kv
+			}
+			switch k {
+			case "filename":
+				fss.filename = v
+			case "name":
+				fss.name = v
+			default:
+				return fmt.Errorf("unknown configuration key for file secret source %q", k)
+			}
+		}
+	}
+	return nil
+}
+
+type fileSecretSource struct {
+	internal map[string]string
+	name     string
+	filename string
+}
+
+func (fss *fileSecretSource) Name() string {
+	return fss.name
+}
+
+func (fss *fileSecretSource) Description() string {
+	return fmt.Sprintf("file source from %s", fss.filename)
+}
+
+func (fss *fileSecretSource) Get(key string) (string, error) {
+	v, ok := fss.internal[key]
+	if !ok {
+		return "", errors.New("no value")
+	}
+	return v, nil
+}

internal/secretsource/init.go

@@ -0,0 +1,7 @@
+// Package secretsource registers all the internal secret sources when imported
+package secretsource
+
+import (
+	_ "go.k6.io/k6/internal/secretsource/file" // import them for init
+	_ "go.k6.io/k6/internal/secretsource/mock" // import them for init
+)

internal/secretsource/mock/mock.go

@@ -0,0 +1,60 @@
+// Package mock implements a secret source that is just taking secrets on the cli
+package mock
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.k6.io/k6/secretsource"
+)
+
+func init() {
+	secretsource.RegisterExtension("mock", func(params secretsource.Params) (secretsource.SecretSource, error) {
+		list := strings.Split(params.ConfigArgument, ",")
+		secrets := make(map[string]string, len(list))
+		name := "mock"
+		for _, kv := range list {
+			k, v, ok := strings.Cut(kv, "=")
+			if !ok {
+				return nil, fmt.Errorf("parsing %q, needs =", kv)
+			}
+			if k == "name" {
+				name = k
+				continue
+			}
+
+			secrets[k] = v
+		}
+		return NewMockSecretSource(name, secrets), nil
+	})
+}
+
+// NewMockSecretSource returns a new secret source mock with the provided name and map of secrets
+func NewMockSecretSource(name string, secrets map[string]string) secretsource.SecretSource {
+	return &mockSecretSource{
+		internal: secrets,
+		name:     name,
+	}
+}
+
+type mockSecretSource struct {
+	internal map[string]string
+	name     string
+}
+
+func (mss *mockSecretSource) Name() string {
+	return mss.name
+}
+
+func (mss *mockSecretSource) Description() string {
+	return "this is a mock secret source"
+}
+
+func (mss *mockSecretSource) Get(key string) (string, error) {
+	v, ok := mss.internal[key]
+	if !ok {
+		return "", errors.New("no value")
+	}
+	return v, nil
+}