compose: add experimental envsubst support (#4477)

Add primitive support for envsubst expressions configuration.

Contributes to #4397,  Azure/azure-dev-pr#1682

Author: weikanglim

cli/azd/.vscode/cspell-azd-dictionary.txt

@@ -95,6 +95,7 @@ endregion
 entraid
 envlist
 envname
+envsubst
 errcheck
 errorinfo
 errorlint
@@ -200,6 +201,7 @@ stdouttrace
 STRINGSLICE
 struct
 structs
+subst
 substr
 swacli
 Syncer

cli/azd/pkg/project/scaffold_gen.go

@@ -154,7 +154,7 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) {
 				Port: -1,
 			}
 
-			err := mapContainerApp(res, &svcSpec)
+			err := mapContainerApp(res, &svcSpec, &infraSpec)
 			if err != nil {
 				return nil, err
 			}
@@ -203,17 +203,36 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) {
 	return &infraSpec, nil
 }
 
-func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec) error {
+func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSpec *scaffold.InfraSpec) error {
 	props := res.Props.(ContainerAppProps)
 	for _, envVar := range props.Env {
+		if len(envVar.Value) == 0 && len(envVar.Secret) == 0 {
+			return fmt.Errorf(
+				"environment variable %s for host %s is invalid: both value and secret are empty",
+				envVar.Name,
+				res.Name)
+		}
+
+		if len(envVar.Value) > 0 && len(envVar.Secret) > 0 {
+			return fmt.Errorf(
+				"environment variable %s for host %s is invalid: both value and secret are set",
+				envVar.Name,
+				res.Name)
+		}
+
 		isSecret := len(envVar.Secret) > 0
 		value := envVar.Value
 		if isSecret {
-			// TODO: handle secrets
-			continue
+			value = envVar.Secret
 		}
 
-		svcSpec.Env[envVar.Name] = value
+		// Notice that we derive isSecret from its usage.
+		// This is generally correct, except for the case where:
+		// - CONNECTION_STRING: ${DB_HOST}:${DB_SECRET}
+		// Here, DB_HOST is not a secret, but DB_SECRET is. And yet, DB_HOST will be marked as a secret.
+		// This is a limitation of the current implementation, but it's safer to mark both as secrets above.
+		evaluatedValue := genBicepParamsFromEnvSubst(value, isSecret, infraSpec)
+		svcSpec.Env[envVar.Name] = evaluatedValue
 	}
 
 	port := props.Port
@@ -256,3 +275,73 @@ func mapHostUses(
 
 	return nil
 }
+
+func setParameter(spec *scaffold.InfraSpec, name string, value string, isSecret bool) {
+	for _, parameters := range spec.Parameters {
+		if parameters.Name == name { // handle existing parameter
+			if isSecret && !parameters.Secret {
+				// escalate the parameter to a secret
+				parameters.Secret = true
+			}
+
+			// prevent auto-generated parameters from being overwritten with different values
+			if valStr, ok := parameters.Value.(string); !ok || ok && valStr != value {
+				// if you are a maintainer and run into this error, consider using a different, unique name
+				panic(fmt.Sprintf(
+					"parameter collision: parameter %s already set to %s, cannot set to %s", name, parameters.Value, value))
+			}
+
+			return
+		}
+	}
+
+	spec.Parameters = append(spec.Parameters, scaffold.Parameter{
+		Name:   name,
+		Value:  value,
+		Type:   "string",
+		Secret: isSecret,
+	})
+}
+
+// genBicepParamsFromEnvSubst generates Bicep input parameters from a string containing envsubst expression(s),
+// returning the substituted string that references these parameters.
+//
+// If the string is a literal, it is returned as is.
+// If isSecret is true, the parameter is marked as a secret.
+func genBicepParamsFromEnvSubst(
+	s string,
+	isSecret bool,
+	infraSpec *scaffold.InfraSpec) string {
+	names, locations := parseEnvSubstVariables(s)
+
+	// add all expressions as parameters
+	for i, name := range names {
+		expression := s[locations[i].start : locations[i].stop+1]
+		setParameter(infraSpec, scaffold.BicepName(name), expression, isSecret)
+	}
+
+	var result string
+	if len(names) == 0 {
+		// literal string with no expressions, quote the value as a Bicep string
+		result = "'" + s + "'"
+	} else if len(names) == 1 {
+		// single expression, return the bicep parameter name to reference the expression
+		result = scaffold.BicepName(names[0])
+	} else {
+		// multiple expressions
+		// construct the string with all expressions replaced by parameter references as a Bicep interpolated string
+		previous := 0
+		result = "'"
+		for i, loc := range locations {
+			// replace each expression with references by variable name
+			result += s[previous:loc.start]
+			result += "${"
+			result += scaffold.BicepName(names[i])
+			result += "}"
+			previous = loc.stop + 1
+		}
+		result += "'"
+	}
+
+	return result
+}

cli/azd/pkg/project/scaffold_gen_test.go

@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package project
+
+import (
+	"testing"
+
+	"github.com/azure/azure-dev/cli/azd/internal/scaffold"
+)
+
+func Test_genBicepParamsFromEnvSubst(t *testing.T) {
+	tests := []struct {
+		// input
+		value         string
+		valueIsSecret bool
+		// output
+		want       string
+		wantParams []scaffold.Parameter
+	}{
+		{"foo", false, "'foo'", nil},
+		{"${MY_VAR}", false, "myVar", []scaffold.Parameter{{Name: "myVar", Value: "${MY_VAR}", Type: "string"}}},
+
+		{"${MY_SECRET}", true, "mySecret",
+			[]scaffold.Parameter{
+				{Name: "mySecret", Value: "${MY_SECRET}", Type: "string", Secret: true}}},
+
+		{"Hello, ${world:=okay}!", false, "world",
+			[]scaffold.Parameter{
+				{Name: "world", Value: "${world:=okay}", Type: "string"}}},
+
+		{"${CAT} and ${DOG}", false, "'${cat} and ${dog}'",
+			[]scaffold.Parameter{
+				{Name: "cat", Value: "${CAT}", Type: "string"},
+				{Name: "dog", Value: "${DOG}", Type: "string"}}},
+
+		{"${DB_HOST:='local'}:${DB_USERNAME:='okay'}", true, "'${dbHost}:${dbUsername}'",
+			[]scaffold.Parameter{
+				{Name: "dbHost", Value: "${DB_HOST:='local'}", Type: "string", Secret: true},
+				{Name: "dbUsername", Value: "${DB_USERNAME:='okay'}", Type: "string", Secret: true}}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.value, func(t *testing.T) {
+			spec := &scaffold.InfraSpec{}
+			evaluated := genBicepParamsFromEnvSubst(tt.value, tt.valueIsSecret, spec)
+			if tt.want != evaluated {
+				t.Errorf("evalEnvValue() evaluatedValue = %v, want %v", evaluated, tt.want)
+			}
+
+			for i, param := range tt.wantParams {
+				found := false
+				for _, generated := range spec.Parameters {
+					if generated.Name == param.Name {
+						if generated.Secret != param.Secret {
+							t.Errorf("evalEnvValue() secret = %v, want %v", generated.Secret, param.Secret)
+						}
+
+						if generated.Value != param.Value {
+							t.Errorf("evalEnvValue() value = %v, want %v", generated.Value, param.Value)
+						}
+
+						if generated.Type != param.Type {
+							t.Errorf("evalEnvValue() type = %v, want %v", generated.Type, param.Type)
+						}
+						found = true
+						break
+					}
+				}
+
+				if !found {
+					t.Errorf("evalEnvValue() parameter = %v not found", spec.Parameters[i].Name)
+				}
+			}
+		})
+	}
+}

cli/azd/pkg/project/scaffold_gen_util.go

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package project
+
+import (
+	"strings"
+	"unicode"
+)
+
+type location struct {
+	start int
+	stop  int
+}
+
+// parseEnvSubstVariables parses the envsubst expression(s) present in a string.
+// substitutions, returning the locations of the expressions and the names of the variables.
+//
+// It works with both:
+//   - ${var} and
+//   - ${var:=default} syntaxes
+func parseEnvSubstVariables(s string) (names []string, expressions []location) {
+	inVar := false
+	inVarName := false
+	name := strings.Builder{}
+
+	i := 0
+	start := 0 // start of the variable expression
+	for i < len(s) {
+		if s[i] == '$' && i+1 < len(s) && s[i+1] == '{' { // detect ${ sequence
+			inVar = true
+			inVarName = true
+			start = i
+			i += len("${")
+			continue
+		}
+
+		if inVar {
+			if inVarName { // detect the end of the variable name
+				// a variable name can contain letters, digits, and underscores, and nothing else.
+				if unicode.IsLetter(rune(s[i])) || unicode.IsDigit(rune(s[i])) || s[i] == '_' {
+					_ = name.WriteByte(s[i])
+				} else { // a non-matching character means we've reached the end of the name
+					inVarName = false
+				}
+			}
+
+			if s[i] == '}' { // detect the end of the variable expression
+				inVar = false
+				names = append(names, name.String())
+				name.Reset()
+				expressions = append(expressions, location{start, i})
+			}
+		}
+
+		i++
+	}
+	return
+}

cli/azd/pkg/project/scaffold_gen_util_test.go

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package project
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_parseEnvSubtVariables(t *testing.T) {
+	tests := []struct {
+		name            string
+		input           string
+		wantNames       []string
+		wantExpressions []location
+	}{
+		{"empty", "", nil, nil},
+		{"no variables", "foo", nil, nil},
+		{"one variable", "${foo}", []string{"foo"}, []location{{0, 5}}},
+		{"two variables", "${foo} ${bar}", []string{"foo", "bar"}, []location{{0, 5}, {7, 12}}},
+		{"two variables with text", "${foo:=value} ${bar#subs}", []string{"foo", "bar"}, []location{{0, 12}, {14, 24}}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotNames, gotExpressions := parseEnvSubstVariables(tt.input)
+			if !reflect.DeepEqual(gotNames, tt.wantNames) {
+				t.Errorf("parseEnvSubtVariables() gotNames = %v, want %v", gotNames, tt.wantNames)
+			}
+			if !reflect.DeepEqual(gotExpressions, tt.wantExpressions) {
+				t.Errorf("parseEnvSubtVariables() gotExpressions = %v, want %v", gotExpressions, tt.wantExpressions)
+			}
+		})
+	}
+}