Skip to content

Commit fa2891a

Browse files
authoredNov 20, 2024··
Merge pull request #855 from Azure/create-schema-validation
Add schema validation for templating configuration
2 parents ebf5a91 + 114539f commit fa2891a

File tree

11 files changed

+724
-4
lines changed

11 files changed

+724
-4
lines changed
 

‎config/config.schema.json

+551
Large diffs are not rendered by default.

‎config/config.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
$schema: config.schema.json
12
defaults:
23
region: {{ .ctx.region }}
34
# Resourcegroups
@@ -207,7 +208,8 @@ clouds:
207208
# DNS
208209
regionalDNSSubdomain: '{{ .ctx.region }}-cs'
209210
# Maestro
210-
maestroRestrictIstioIngress: false
211+
maestro:
212+
restrictIstioIngress: false
211213
personal-dev:
212214
# this is the personal DEV environment
213215
defaults:

‎config/public-cloud-cs-pr.json

+1-2
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,8 @@
6666
"serverStorageSizeGB": "32",
6767
"serverVersion": "15"
6868
},
69-
"restrictIstioIngress": true
69+
"restrictIstioIngress": false
7070
},
71-
"maestroRestrictIstioIngress": false,
7271
"managementClusterSubscription": "ARO Hosted Control Planes (EA Subscription 1)",
7372
"mgmt": {
7473
"etcd": {

‎go.work.sum

+3
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
608608
github.com/digitalocean/godo v1.99.0/go.mod h1:SsS2oXo2rznfM/nORlZ/6JaUJZFhmKTib1YhopUc8NA=
609609
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
610610
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI=
611+
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
611612
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
612613
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
613614
github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -742,6 +743,7 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
742743
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
743744
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
744745
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
746+
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
745747
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
746748
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
747749
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -968,6 +970,7 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS
968970
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
969971
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
970972
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
973+
github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
971974
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
972975
github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8=
973976
github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU=

‎tooling/templatize/go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
3636
github.com/pkg/errors v0.9.1 // indirect
3737
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
38+
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect
3839
github.com/shopspring/decimal v1.4.0 // indirect
3940
github.com/spf13/cast v1.7.0 // indirect
4041
github.com/spf13/pflag v1.0.5 // indirect

‎tooling/templatize/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJ
6969
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
7070
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
7171
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
72+
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw=
73+
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
7274
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
7375
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
7476
github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=

‎tooling/templatize/pkg/config/config.go

+69
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"bytes"
55
"fmt"
66
"os"
7+
"path/filepath"
78
"reflect"
89
"text/template"
910

11+
"github.com/santhosh-tekuri/jsonschema/v6"
1012
"gopkg.in/yaml.v3"
1113
)
1214

@@ -89,6 +91,62 @@ func mergeVariables(base, override Variables) Variables {
8991
return base
9092
}
9193

94+
// Needed to convert Variables to map[string]interface{} for jsonschema validation
95+
// see: https://github.com/santhosh-tekuri/jsonschema/blob/boon/schema.go#L124
96+
func convertToInterface(variables Variables) map[string]any {
97+
m := map[string]any{}
98+
for k, v := range variables {
99+
if subMap, ok := v.(Variables); ok {
100+
m[k] = convertToInterface(subMap)
101+
} else {
102+
m[k] = v
103+
}
104+
}
105+
return m
106+
}
107+
108+
func (cp *configProviderImpl) loadSchema() (any, error) {
109+
schemaPath := cp.schema
110+
if !filepath.IsAbs(schemaPath) {
111+
schemaPath = filepath.Join(filepath.Dir(cp.config), schemaPath)
112+
}
113+
reader, err := os.Open(schemaPath)
114+
if err != nil {
115+
return nil, fmt.Errorf("failed to open schema file: %v", err)
116+
}
117+
118+
schema, err := jsonschema.UnmarshalJSON(reader)
119+
if err != nil {
120+
return nil, fmt.Errorf("failed to unmarshal schema: %v", err)
121+
}
122+
123+
return schema, nil
124+
}
125+
126+
func (cp *configProviderImpl) validateSchema(variables Variables) error {
127+
c := jsonschema.NewCompiler()
128+
129+
schema, err := cp.loadSchema()
130+
if err != nil {
131+
return fmt.Errorf("failed to load schema: %v", err)
132+
}
133+
134+
err = c.AddResource(cp.schema, schema)
135+
if err != nil {
136+
return fmt.Errorf("failed to add schema resource: %v", err)
137+
}
138+
sch, err := c.Compile(cp.schema)
139+
if err != nil {
140+
return fmt.Errorf("failed to compile schema: %v", err)
141+
}
142+
143+
err = sch.Validate(convertToInterface(variables))
144+
if err != nil {
145+
return fmt.Errorf("failed to validate schema: %v", err)
146+
}
147+
return nil
148+
}
149+
92150
func (cp *configProviderImpl) GetVariables(cloud, deployEnv, region string, configReplacements *ConfigReplacements) (Variables, error) {
93151
variables, err := cp.GetDeployEnvVariables(cloud, deployEnv, configReplacements)
94152
if err != nil {
@@ -102,6 +160,11 @@ func (cp *configProviderImpl) GetVariables(cloud, deployEnv, region string, conf
102160
}
103161
mergeVariables(variables, regionOverrides)
104162

163+
// validate schema
164+
err = cp.validateSchema(variables)
165+
if err != nil {
166+
return nil, err
167+
}
105168
return variables, nil
106169
}
107170

@@ -117,6 +180,10 @@ func (cp *configProviderImpl) Validate(cloud, deployEnv string) error {
117180
if ok := config.HasDeployEnv(cloud, deployEnv); !ok {
118181
return fmt.Errorf("the deployment env %s is not found under cloud %s", deployEnv, cloud)
119182
}
183+
184+
if !config.HasSchema() {
185+
return fmt.Errorf("$schema not found in config")
186+
}
120187
return nil
121188
}
122189

@@ -135,6 +202,8 @@ func (cp *configProviderImpl) GetDeployEnvVariables(cloud, deployEnv string, con
135202
mergeVariables(variables, config.GetCloudOverrides(cloud))
136203
mergeVariables(variables, config.GetDeployEnvOverrides(cloud, deployEnv))
137204

205+
cp.schema = config.GetSchema()
206+
138207
return variables, nil
139208
}
140209

‎tooling/templatize/pkg/config/config_test.go

+77
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package config
22

33
import (
44
"fmt"
5+
"os"
56
"testing"
67

78
"github.com/stretchr/testify/assert"
@@ -154,3 +155,79 @@ func TestMergeVariable(t *testing.T) {
154155
}
155156

156157
}
158+
159+
func TestLoadSchema(t *testing.T) {
160+
testDirs := t.TempDir()
161+
162+
err := os.WriteFile(testDirs+"/schema.json", []byte(`{"type": "object"}`), 0644)
163+
assert.Nil(t, err)
164+
165+
configProvider := configProviderImpl{}
166+
configProvider.schema = testDirs + "/schema.json"
167+
168+
schema, err := configProvider.loadSchema()
169+
assert.Nil(t, err)
170+
assert.NotNil(t, schema)
171+
assert.Equal(t, map[string]any{"type": "object"}, schema)
172+
}
173+
174+
func TestLoadSchemaError(t *testing.T) {
175+
testDirs := t.TempDir()
176+
177+
err := os.WriteFile(testDirs+"/schma.json", []byte(`{"type": "object"}`), 0644)
178+
assert.Nil(t, err)
179+
180+
configProvider := configProviderImpl{}
181+
configProvider.schema = testDirs + "/schema.json"
182+
_, err = configProvider.loadSchema()
183+
assert.NotNil(t, err)
184+
}
185+
186+
func TestValidateSchema(t *testing.T) {
187+
testSchema := `{
188+
"type": "object",
189+
"properties": {
190+
"key1": {
191+
"type": "string"
192+
}
193+
},
194+
"additionalProperties": false
195+
}`
196+
197+
testDirs := t.TempDir()
198+
199+
err := os.WriteFile(testDirs+"/schema.json", []byte(testSchema), 0644)
200+
assert.Nil(t, err)
201+
202+
configProvider := configProviderImpl{}
203+
configProvider.schema = "schema.json"
204+
configProvider.config = testDirs + "/config.yaml"
205+
206+
err = configProvider.validateSchema(map[string]any{"foo": "bar"})
207+
assert.NotNil(t, err)
208+
assert.ErrorContains(t, err, "additional properties 'foo' not allowed")
209+
210+
err = configProvider.validateSchema(map[string]any{"key1": "bar"})
211+
assert.Nil(t, err)
212+
}
213+
214+
func TestConvertToInterface(t *testing.T) {
215+
vars := Variables{
216+
"key1": "value1",
217+
"key2": Variables{
218+
"key3": "value3",
219+
},
220+
}
221+
222+
expected := map[string]any{
223+
"key1": "value1",
224+
"key2": map[string]any{
225+
"key3": "value3",
226+
},
227+
}
228+
229+
result := convertToInterface(vars)
230+
assert.Equal(t, expected, result)
231+
assert.IsType(t, expected, map[string]any{})
232+
assert.IsType(t, expected["key2"], map[string]any{})
233+
}

‎tooling/templatize/pkg/config/types.go

+13-1
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@ import (
66

77
type configProviderImpl struct {
88
config string
9+
schema string
910
}
1011

11-
type Variables map[string]interface{}
12+
type Variables map[string]any
1213

1314
func NewVariableOverrides() VariableOverrides {
1415
return &variableOverrides{}
@@ -20,11 +21,14 @@ type VariableOverrides interface {
2021
GetDeployEnvOverrides(cloud, deployEnv string) Variables
2122
GetRegionOverrides(cloud, deployEnv, region string) Variables
2223
GetRegions(cloud, deployEnv string) []string
24+
GetSchema() string
25+
HasSchema() bool
2326
HasCloud(cloud string) bool
2427
HasDeployEnv(cloud, deployEnv string) bool
2528
}
2629

2730
type variableOverrides struct {
31+
Schema string `yaml:"$schema"`
2832
Defaults Variables `yaml:"defaults"`
2933
// key is the cloud alias
3034
Overrides map[string]*struct {
@@ -38,6 +42,14 @@ type variableOverrides struct {
3842
} `yaml:"clouds"`
3943
}
4044

45+
func (vo *variableOverrides) GetSchema() string {
46+
return vo.Schema
47+
}
48+
49+
func (vo *variableOverrides) HasSchema() bool {
50+
return vo.Schema != ""
51+
}
52+
4153
func (vo *variableOverrides) GetDefaults() Variables {
4254
return vo.Defaults
4355
}

‎tooling/templatize/testdata/config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
$schema: schema.json
12
defaults:
23
region: {{ .ctx.region }}
34
serviceClusterSubscription: hcp-{{ .ctx.region }}
+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "object"
3+
}

0 commit comments

Comments
 (0)
Please sign in to comment.