From b3299e96cfd22384b1c43a1993312efa89c83b8f Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Fri, 1 Nov 2024 14:38:42 -0700 Subject: [PATCH] compose: add experimental envsubst support (#4477) Add primitive support for envsubst expressions configuration. Contributes to #4397, https://github.com/Azure/azure-dev-pr/issues/1682 --- cli/azd/.vscode/cspell-azd-dictionary.txt | 2 + cli/azd/pkg/project/scaffold_gen.go | 99 ++++++++++++++++++- cli/azd/pkg/project/scaffold_gen_test.go | 76 ++++++++++++++ cli/azd/pkg/project/scaffold_gen_util.go | 59 +++++++++++ cli/azd/pkg/project/scaffold_gen_util_test.go | 35 +++++++ 5 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 cli/azd/pkg/project/scaffold_gen_test.go create mode 100644 cli/azd/pkg/project/scaffold_gen_util.go create mode 100644 cli/azd/pkg/project/scaffold_gen_util_test.go diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt index a1da38a9dbd..9ee767db8b4 100644 --- a/cli/azd/.vscode/cspell-azd-dictionary.txt +++ b/cli/azd/.vscode/cspell-azd-dictionary.txt @@ -95,6 +95,7 @@ endregion entraid envlist envname +envsubst errcheck errorinfo errorlint @@ -200,6 +201,7 @@ stdouttrace STRINGSLICE struct structs +subst substr swacli Syncer diff --git a/cli/azd/pkg/project/scaffold_gen.go b/cli/azd/pkg/project/scaffold_gen.go index 0f3d1e6dd3b..595d5f14b8c 100644 --- a/cli/azd/pkg/project/scaffold_gen.go +++ b/cli/azd/pkg/project/scaffold_gen.go @@ -154,7 +154,7 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { Port: -1, } - err := mapContainerApp(res, &svcSpec) + err := mapContainerApp(res, &svcSpec, &infraSpec) if err != nil { return nil, err } @@ -203,17 +203,36 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { return &infraSpec, nil } -func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec) error { +func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSpec *scaffold.InfraSpec) error { props := res.Props.(ContainerAppProps) for _, envVar := range props.Env { + if len(envVar.Value) == 0 && len(envVar.Secret) == 0 { + return fmt.Errorf( + "environment variable %s for host %s is invalid: both value and secret are empty", + envVar.Name, + res.Name) + } + + if len(envVar.Value) > 0 && len(envVar.Secret) > 0 { + return fmt.Errorf( + "environment variable %s for host %s is invalid: both value and secret are set", + envVar.Name, + res.Name) + } + isSecret := len(envVar.Secret) > 0 value := envVar.Value if isSecret { - // TODO: handle secrets - continue + value = envVar.Secret } - svcSpec.Env[envVar.Name] = value + // Notice that we derive isSecret from its usage. + // This is generally correct, except for the case where: + // - CONNECTION_STRING: ${DB_HOST}:${DB_SECRET} + // Here, DB_HOST is not a secret, but DB_SECRET is. And yet, DB_HOST will be marked as a secret. + // This is a limitation of the current implementation, but it's safer to mark both as secrets above. + evaluatedValue := genBicepParamsFromEnvSubst(value, isSecret, infraSpec) + svcSpec.Env[envVar.Name] = evaluatedValue } port := props.Port @@ -256,3 +275,73 @@ func mapHostUses( return nil } + +func setParameter(spec *scaffold.InfraSpec, name string, value string, isSecret bool) { + for _, parameters := range spec.Parameters { + if parameters.Name == name { // handle existing parameter + if isSecret && !parameters.Secret { + // escalate the parameter to a secret + parameters.Secret = true + } + + // prevent auto-generated parameters from being overwritten with different values + if valStr, ok := parameters.Value.(string); !ok || ok && valStr != value { + // if you are a maintainer and run into this error, consider using a different, unique name + panic(fmt.Sprintf( + "parameter collision: parameter %s already set to %s, cannot set to %s", name, parameters.Value, value)) + } + + return + } + } + + spec.Parameters = append(spec.Parameters, scaffold.Parameter{ + Name: name, + Value: value, + Type: "string", + Secret: isSecret, + }) +} + +// genBicepParamsFromEnvSubst generates Bicep input parameters from a string containing envsubst expression(s), +// returning the substituted string that references these parameters. +// +// If the string is a literal, it is returned as is. +// If isSecret is true, the parameter is marked as a secret. +func genBicepParamsFromEnvSubst( + s string, + isSecret bool, + infraSpec *scaffold.InfraSpec) string { + names, locations := parseEnvSubstVariables(s) + + // add all expressions as parameters + for i, name := range names { + expression := s[locations[i].start : locations[i].stop+1] + setParameter(infraSpec, scaffold.BicepName(name), expression, isSecret) + } + + var result string + if len(names) == 0 { + // literal string with no expressions, quote the value as a Bicep string + result = "'" + s + "'" + } else if len(names) == 1 { + // single expression, return the bicep parameter name to reference the expression + result = scaffold.BicepName(names[0]) + } else { + // multiple expressions + // construct the string with all expressions replaced by parameter references as a Bicep interpolated string + previous := 0 + result = "'" + for i, loc := range locations { + // replace each expression with references by variable name + result += s[previous:loc.start] + result += "${" + result += scaffold.BicepName(names[i]) + result += "}" + previous = loc.stop + 1 + } + result += "'" + } + + return result +} diff --git a/cli/azd/pkg/project/scaffold_gen_test.go b/cli/azd/pkg/project/scaffold_gen_test.go new file mode 100644 index 00000000000..85cf4125075 --- /dev/null +++ b/cli/azd/pkg/project/scaffold_gen_test.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/internal/scaffold" +) + +func Test_genBicepParamsFromEnvSubst(t *testing.T) { + tests := []struct { + // input + value string + valueIsSecret bool + // output + want string + wantParams []scaffold.Parameter + }{ + {"foo", false, "'foo'", nil}, + {"${MY_VAR}", false, "myVar", []scaffold.Parameter{{Name: "myVar", Value: "${MY_VAR}", Type: "string"}}}, + + {"${MY_SECRET}", true, "mySecret", + []scaffold.Parameter{ + {Name: "mySecret", Value: "${MY_SECRET}", Type: "string", Secret: true}}}, + + {"Hello, ${world:=okay}!", false, "world", + []scaffold.Parameter{ + {Name: "world", Value: "${world:=okay}", Type: "string"}}}, + + {"${CAT} and ${DOG}", false, "'${cat} and ${dog}'", + []scaffold.Parameter{ + {Name: "cat", Value: "${CAT}", Type: "string"}, + {Name: "dog", Value: "${DOG}", Type: "string"}}}, + + {"${DB_HOST:='local'}:${DB_USERNAME:='okay'}", true, "'${dbHost}:${dbUsername}'", + []scaffold.Parameter{ + {Name: "dbHost", Value: "${DB_HOST:='local'}", Type: "string", Secret: true}, + {Name: "dbUsername", Value: "${DB_USERNAME:='okay'}", Type: "string", Secret: true}}}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + spec := &scaffold.InfraSpec{} + evaluated := genBicepParamsFromEnvSubst(tt.value, tt.valueIsSecret, spec) + if tt.want != evaluated { + t.Errorf("evalEnvValue() evaluatedValue = %v, want %v", evaluated, tt.want) + } + + for i, param := range tt.wantParams { + found := false + for _, generated := range spec.Parameters { + if generated.Name == param.Name { + if generated.Secret != param.Secret { + t.Errorf("evalEnvValue() secret = %v, want %v", generated.Secret, param.Secret) + } + + if generated.Value != param.Value { + t.Errorf("evalEnvValue() value = %v, want %v", generated.Value, param.Value) + } + + if generated.Type != param.Type { + t.Errorf("evalEnvValue() type = %v, want %v", generated.Type, param.Type) + } + found = true + break + } + } + + if !found { + t.Errorf("evalEnvValue() parameter = %v not found", spec.Parameters[i].Name) + } + } + }) + } +} diff --git a/cli/azd/pkg/project/scaffold_gen_util.go b/cli/azd/pkg/project/scaffold_gen_util.go new file mode 100644 index 00000000000..26a1934750c --- /dev/null +++ b/cli/azd/pkg/project/scaffold_gen_util.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "strings" + "unicode" +) + +type location struct { + start int + stop int +} + +// parseEnvSubstVariables parses the envsubst expression(s) present in a string. +// substitutions, returning the locations of the expressions and the names of the variables. +// +// It works with both: +// - ${var} and +// - ${var:=default} syntaxes +func parseEnvSubstVariables(s string) (names []string, expressions []location) { + inVar := false + inVarName := false + name := strings.Builder{} + + i := 0 + start := 0 // start of the variable expression + for i < len(s) { + if s[i] == '$' && i+1 < len(s) && s[i+1] == '{' { // detect ${ sequence + inVar = true + inVarName = true + start = i + i += len("${") + continue + } + + if inVar { + if inVarName { // detect the end of the variable name + // a variable name can contain letters, digits, and underscores, and nothing else. + if unicode.IsLetter(rune(s[i])) || unicode.IsDigit(rune(s[i])) || s[i] == '_' { + _ = name.WriteByte(s[i]) + } else { // a non-matching character means we've reached the end of the name + inVarName = false + } + } + + if s[i] == '}' { // detect the end of the variable expression + inVar = false + names = append(names, name.String()) + name.Reset() + expressions = append(expressions, location{start, i}) + } + } + + i++ + } + return +} diff --git a/cli/azd/pkg/project/scaffold_gen_util_test.go b/cli/azd/pkg/project/scaffold_gen_util_test.go new file mode 100644 index 00000000000..347fff311bd --- /dev/null +++ b/cli/azd/pkg/project/scaffold_gen_util_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "reflect" + "testing" +) + +func Test_parseEnvSubtVariables(t *testing.T) { + tests := []struct { + name string + input string + wantNames []string + wantExpressions []location + }{ + {"empty", "", nil, nil}, + {"no variables", "foo", nil, nil}, + {"one variable", "${foo}", []string{"foo"}, []location{{0, 5}}}, + {"two variables", "${foo} ${bar}", []string{"foo", "bar"}, []location{{0, 5}, {7, 12}}}, + {"two variables with text", "${foo:=value} ${bar#subs}", []string{"foo", "bar"}, []location{{0, 12}, {14, 24}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotNames, gotExpressions := parseEnvSubstVariables(tt.input) + if !reflect.DeepEqual(gotNames, tt.wantNames) { + t.Errorf("parseEnvSubtVariables() gotNames = %v, want %v", gotNames, tt.wantNames) + } + if !reflect.DeepEqual(gotExpressions, tt.wantExpressions) { + t.Errorf("parseEnvSubtVariables() gotExpressions = %v, want %v", gotExpressions, tt.wantExpressions) + } + }) + } +}