From 08b00c72f19bebccc3369da313f68ebbde6fbd9d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 08:36:07 +0200 Subject: [PATCH 01/48] chore(deps): update dependency golangci/golangci-lint to v1.59.0 (#6090) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .tools_versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tools_versions.yaml b/.tools_versions.yaml index ec0378e39b..3c1da690b8 100644 --- a/.tools_versions.yaml +++ b/.tools_versions.yaml @@ -5,7 +5,7 @@ controller-tools: "0.15.0" # renovate: datasource=github-releases depName=kubernetes-sigs/kustomize kustomize: "5.3.0" # renovate: datasource=github-releases depName=golangci/golangci-lint -golangci-lint: "1.58.2" +golangci-lint: "1.59.0" # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold skaffold: "2.12.0" # renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime From 78900c70546ab18877ce25512626fefc43697522 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 08:36:50 +0200 Subject: [PATCH 02/48] chore(deps): bump github.com/kong/go-database-reconciler (#6085) Bumps [github.com/kong/go-database-reconciler](https://github.com/kong/go-database-reconciler) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/kong/go-database-reconciler/releases) - [Commits](https://github.com/kong/go-database-reconciler/compare/v1.10.0...v1.11.0) --- updated-dependencies: - dependency-name: github.com/kong/go-database-reconciler dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index f7b8d106b7..6380c1012d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/jpillora/backoff v1.0.0 - github.com/kong/go-database-reconciler v1.10.0 + github.com/kong/go-database-reconciler v1.11.0 github.com/kong/go-kong v0.55.0 github.com/kong/kubernetes-telemetry v0.1.3 github.com/kong/kubernetes-testing-framework v0.47.0 @@ -175,7 +175,7 @@ require ( github.com/prometheus/procfs v0.14.0 // indirect github.com/puzpuzpuz/xsync/v2 v2.5.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil/v3 v3.24.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/go.sum b/go.sum index c55f94d94c..01bf373348 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kong/go-database-reconciler v1.10.0 h1:502Nn7CTsUNZyClL5bjrhkFrendVrPxDkP4ZGlBfsdg= -github.com/kong/go-database-reconciler v1.10.0/go.mod h1:88/u23NIhkQr7SeTP1p4nLcnWCm8AMCwCF1f7Telw+w= +github.com/kong/go-database-reconciler v1.11.0 h1:AowvG85lOWMZSCMZ9UpCGLAYoq/lXvoXJGc6BXH2+qU= +github.com/kong/go-database-reconciler v1.11.0/go.mod h1:fX9SV2ukbuUdVw2h1rakWTi/DUxAV9cbj4QWttoiRrc= github.com/kong/go-kong v0.55.0 h1:lonKRzsDGk12dh9E+y+pWnY2ThXhKuMHjzBHSpCvQLw= github.com/kong/go-kong v0.55.0/go.mod h1:i1cMgTu6RYPHSyMpviShddRnc+DML/vlpgKC00hr8kU= github.com/kong/kubernetes-telemetry v0.1.3 h1:Hz2tkHGIIUqbn1x46QRDmmNjbEtJyxyOvHSPne3uPto= @@ -376,8 +376,8 @@ github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.3.0 h1:OLFHZ91Z7NiNP3dnaPxLxCDXlb6TBuxFzMvv6bu+Ptw= github.com/sethvargo/go-password v0.3.0/go.mod h1:p6we8DZ0eyYXof9pon7Cqrw98N4KTaYiadDml1dUEEw= -github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= -github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -548,7 +548,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 748ff85316b2de0191531b63225d9714681a8369 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 08:37:45 +0200 Subject: [PATCH 03/48] chore(deps): bump google.golang.org/api from 0.180.0 to 0.181.0 (#6051) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.180.0 to 0.181.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.180.0...v0.181.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 6380c1012d..c47b045308 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 go.uber.org/zap v1.27.0 - google.golang.org/api v0.180.0 + google.golang.org/api v0.181.0 k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.30.1 k8s.io/apimachinery v0.30.1 @@ -215,7 +215,7 @@ require ( golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index 01bf373348..0754b6e3c7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= -cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= +cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= +cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= @@ -583,8 +583,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.180.0 h1:M2D87Yo0rGBPWpo1orwfCLehUUL6E7/TYe5gvMQWDh4= -google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= +google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4= +google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -592,8 +592,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae h1:c55+MER4zkBS14uJhSZMGGmya0yJx5iHV4x/fpOSNRk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= From a7c0e93b5a2651eb3f3feba05a75d3e598a064fe Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Mon, 27 May 2024 01:05:27 -0700 Subject: [PATCH 04/48] fix(diag) correct redaction toggle (#6073) --- CHANGELOG.md | 2 + internal/dataplane/kong_client.go | 4 +- internal/dataplane/kong_client_test.go | 66 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4f1c29f1..5a4ea65c74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -179,6 +179,8 @@ Adding a new version? You'll need three changes: [#5919](https://github.com/Kong/kubernetes-ingress-controller/pull/5919) - Redacted values no longer cause collisions in configuration reported to Konnect. [5964](https://github.com/Kong/kubernetes-ingress-controller/pull/5964) +- The `--dump-sensitive-config` flag is no longer backwards. + [6073](https://github.com/Kong/kubernetes-ingress-controller/pull/6073) ### Changed diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 0d738b1027..91435bf138 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -759,14 +759,14 @@ func prepareSendDiagnosticFn( var config *file.Content if diagnosticConfig.DumpsIncludeSensitive { + config = targetContent + } else { redactedConfig := deckgen.ToDeckContent(ctx, logger, targetState.SanitizedCopy(util.DefaultUUIDGenerator{}), deckGenParams, ) config = redactedConfig - } else { - config = targetContent } return func(failed bool, rawResponseBody []byte) { diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index f036414670..27858aa95a 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -1156,3 +1156,69 @@ func cacheStoresFromObjs(t *testing.T, objs ...runtime.Object) store.CacheStores require.NoError(t, err) return s } + +func TestKongClient_ConfigDumpSanitization(t *testing.T) { + clientsProvider := mockGatewayClientsProvider{ + gatewayClients: []*adminapi.Client{ + mustSampleGatewayClient(t), + }, + konnectClient: mustSampleKonnectClient(t), + } + updateStrategyResolver := newMockUpdateStrategyResolver(t) + configChangeDetector := mockConfigurationChangeDetector{hasConfigurationChanged: true} + configBuilder := newMockKongConfigBuilder() + kongRawStateGetter := &mockKongLastValidConfigFetcher{} + + const testPrivateKey = "private-key-string" + configBuilder.kongState = &kongstate.KongState{ + Certificates: []kongstate.Certificate{ + { + Certificate: kong.Certificate{ + ID: kong.String("new_cert"), + Key: kong.String(testPrivateKey), // This should be redacted. + }, + }, + }, + } + kongClient := setupTestKongClient(t, updateStrategyResolver, clientsProvider, configChangeDetector, configBuilder, nil, kongRawStateGetter) + + testCases := []struct { + name string + dumpsIncludeSensitive bool + expectSanitizedDump bool + }{ + { + name: "when DumpsIncludeSensitive is true, expect no sanitization", + dumpsIncludeSensitive: true, + expectSanitizedDump: false, + }, + { + name: "when DumpsIncludeSensitive is false, expect sanitization", + dumpsIncludeSensitive: false, + expectSanitizedDump: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + diagnosticsCh := make(chan util.ConfigDump, 1) // make it buffered to avoid blocking + kongClient.diagnostic = util.ConfigDumpDiagnostic{ + Configs: diagnosticsCh, + DumpsIncludeSensitive: tc.dumpsIncludeSensitive, + } + ctx := context.Background() + err := kongClient.Update(ctx) + require.NoError(t, err) + + dump := <-diagnosticsCh + require.NotNil(t, dump.Config) + require.Len(t, dump.Config.Certificates, 1) + dumpedCert := dump.Config.Certificates[0] + if tc.expectSanitizedDump { + require.Equal(t, "{vault://redacted-value}", *dumpedCert.Key) + } else { + require.Equal(t, testPrivateKey, *dumpedCert.Key) + } + }) + } +} From 6df0a758bba844b3ca4d3fcc9829c3f0326f2e7f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 10:35:29 +0200 Subject: [PATCH 05/48] chore(deps): update dependency kubernetes-sigs/controller-runtime to v0.18.3 (#6089) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .tools_versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tools_versions.yaml b/.tools_versions.yaml index 3c1da690b8..2b867be195 100644 --- a/.tools_versions.yaml +++ b/.tools_versions.yaml @@ -9,7 +9,7 @@ golangci-lint: "1.59.0" # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold skaffold: "2.12.0" # renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime -setup-envtest: "0.18.2" +setup-envtest: "0.18.3" # renovate: datasource=github-releases depName=elastic/crd-ref-docs crd-ref-docs: "0.0.12" # renovate: datasource=github-releases depName=mikefarah/yq From 183519e63e2dbca6228c437bfd1bb656fb7e9b03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 12:48:56 +0200 Subject: [PATCH 06/48] chore(deps): bump sigs.k8s.io/kustomize/api from 0.17.1 to 0.17.2 (#6078) Bumps [sigs.k8s.io/kustomize/api](https://github.com/kubernetes-sigs/kustomize) from 0.17.1 to 0.17.2. - [Release notes](https://github.com/kubernetes-sigs/kustomize/releases) - [Commits](https://github.com/kubernetes-sigs/kustomize/compare/api/v0.17.1...api/v0.17.2) --- updated-dependencies: - dependency-name: sigs.k8s.io/kustomize/api dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c47b045308..bfcbc45000 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( k8s.io/component-base v0.30.1 sigs.k8s.io/controller-runtime v0.18.2 sigs.k8s.io/gateway-api v1.1.0 - sigs.k8s.io/kustomize/api v0.17.1 + sigs.k8s.io/kustomize/api v0.17.2 sigs.k8s.io/kustomize/kyaml v0.17.1 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 0754b6e3c7..ac6afa30df 100644 --- a/go.sum +++ b/go.sum @@ -667,8 +667,8 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.22.0 h1:z/+yr/azoOfzsfooqRsPw1wjJlqT/ukXP0ShkHwNlsI= sigs.k8s.io/kind v0.22.0/go.mod h1:aBlbxg08cauDgZ612shr017/rZwqd7AS563FvpWKPVs= -sigs.k8s.io/kustomize/api v0.17.1 h1:MYJBOP/yQ3/5tp4/sf6HiiMfNNyO97LmtnirH9SLNr4= -sigs.k8s.io/kustomize/api v0.17.1/go.mod h1:ffn5491s2EiNrJSmgqcWGzQUVhc/pB0OKNI0HsT/0tA= +sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g= +sigs.k8s.io/kustomize/api v0.17.2/go.mod h1:UWTz9Ct+MvoeQsHcJ5e+vziRRkwimm3HytpZgIYqye0= sigs.k8s.io/kustomize/kyaml v0.17.1 h1:TnxYQxFXzbmNG6gOINgGWQt09GghzgTP6mIurOgrLCQ= sigs.k8s.io/kustomize/kyaml v0.17.1/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= From 00b48664f06d209952104ef7d305c32ceed3a3e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 27 May 2024 15:13:20 +0200 Subject: [PATCH 07/48] fix(gwapi): only clear status of Gateway API *Routes reconciled by KIC's gateway(s) (#6079) * fix(gwapi): only clear status of Gateway API Routes reconciled by KIC's gateway(s) * refactor: fix clearing status for all types of supported Gateway API routes * chore: reword changelog * Apply suggestions from code review Co-authored-by: Jakub Warczarek --------- Co-authored-by: Jakub Warczarek --- CHANGELOG.md | 3 + .../gateway/grpcroute_controller.go | 69 +++----- .../gateway/httproute_controller.go | 83 ++++----- .../gateway/route_parent_status.go | 32 ++++ .../controllers/gateway/route_predicates.go | 108 ++++++++++++ internal/controllers/gateway/route_utils.go | 73 +++++++- .../controllers/gateway/route_utils_test.go | 2 +- .../gateway/tcproute_controller.go | 69 +++----- .../gateway/tlsroute_controller.go | 69 +++----- .../gateway/udproute_controller.go | 68 +++----- internal/gatewayapi/aliases.go | 6 +- test/envtest/httproute_controller_test.go | 161 ++++++++++++++---- test/internal/helpers/gatewayapi.go | 2 +- 13 files changed, 480 insertions(+), 265 deletions(-) create mode 100644 internal/controllers/gateway/route_predicates.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4ea65c74..3fb3961031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -181,6 +181,9 @@ Adding a new version? You'll need three changes: [5964](https://github.com/Kong/kubernetes-ingress-controller/pull/5964) - The `--dump-sensitive-config` flag is no longer backwards. [6073](https://github.com/Kong/kubernetes-ingress-controller/pull/6073) +- Fixed KIC clearing Gateway API *Route status of routes that it shouldn't reconcilce, e.g. + those attached to Gateways that do not belong to GatewayClass that KIC reconciles. + [6079](https://github.com/Kong/kubernetes-ingress-controller/pull/6079) ### Changed diff --git a/internal/controllers/gateway/grpcroute_controller.go b/internal/controllers/gateway/grpcroute_controller.go index c033c57574..6a95ec5537 100644 --- a/internal/controllers/gateway/grpcroute_controller.go +++ b/internal/controllers/gateway/grpcroute_controller.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "reflect" "time" @@ -99,12 +100,28 @@ func (r *GRPCRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { ) } - // because of the additional burden of having to manage reference data-plane - // configurations for GRPCRoute objects in the underlying Kong Gateway, we - // simply reconcile ALL GRPCRoute objects. This allows us to drop the backend - // data-plane config for an GRPCRoute if it somehow becomes disconnected from - // a supported Gateway and GatewayClass. - return blder.For(&gatewayapi.GRPCRoute{}). + // We enqueue only routes that are: + // - attached during creation or deletion + // - have been attached or detached to a reconciled Gateway. + // This allows us to drop the backend data-plane config for a route if + // it somehow becomes disconnected from a supported Gateway and GatewayClass. + return blder. + For(&gatewayapi.GRPCRoute{}, + builder.WithPredicates(predicate.Funcs{ + GenericFunc: func(_ event.GenericEvent) bool { + return false // we don't need to enqueue from generic + }, + CreateFunc: func(e event.CreateEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.GRPCRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isOrWasRouteAttachedToReconciledGateway[*gatewayapi.GRPCRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.GRPCRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + }), + ). Complete(r) } @@ -305,22 +322,14 @@ func (r *GRPCRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( debug(log, grpcroute, "Retrieving GatewayClass and Gateway for route") gateways, err := getSupportedGatewayForRoute(ctx, log, r.Client, grpcroute, r.GatewayNN) if err != nil { - if err.Error() == unsupportedGW { - debug(log, grpcroute, "Unsupported route found, processing to verify whether it was ever supported") + if errors.Is(err, ErrNoSupportedGateway) { // if there's no supported Gateway then this route could have been previously // supported by this controller. As such we ensure that no supported Gateway // references exist in the object status any longer. - statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, grpcroute) - if err != nil { + if _, err := ensureGatewayReferenceStatusRemoved(ctx, r.Client, log, grpcroute); err != nil { // some failure happened so we need to retry to avoid orphaned statuses return ctrl.Result{}, err } - if statusUpdated { - // the status did in fact needed to be updated, so no need to requeue - // as the status update will trigger a requeue. - debug(log, grpcroute, "Unsupported route was previously supported, status was updated") - return ctrl.Result{}, nil - } // if the route doesn't have a supported Gateway+GatewayClass associated with // it it's possible it became orphaned after becoming queued. In either case @@ -523,31 +532,3 @@ func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont // the status needed an update and it was updated successfully return true, nil } - -// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway -// implementation to prune status references to Gateways supported by this controller -// in the provided GRPCRoute object. -func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, grpcroute *gatewayapi.GRPCRoute) (bool, error) { - // drop all status references to supported Gateway objects - newStatuses := make([]gatewayapi.RouteParentStatus, 0) - for _, status := range grpcroute.Status.Parents { - if status.ControllerName != GetControllerName() { - newStatuses = append(newStatuses, status) - } - } - - // if the new list of statuses is the same length as the old - // nothing has changed and we're all done. - if len(newStatuses) == len(grpcroute.Status.Parents) { - return false, nil - } - - // update the object status in the API - grpcroute.Status.Parents = newStatuses - if err := r.Status().Update(ctx, grpcroute); err != nil { - return false, err - } - - // the status needed an update and it was updated successfully - return true, nil -} diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index 614569ac77..454f6f1567 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "reflect" "slices" @@ -106,7 +107,7 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { if r.enableReferenceGrant { blder.Watches(&gatewayapi.ReferenceGrant{}, - handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForHTTPRoute), + handler.EnqueueRequestsFromMapFunc(r.listHTTPRoutesForReferenceGrant), builder.WithPredicates(predicate.NewPredicateFuncs(referenceGrantHasHTTPRouteFrom)), ) } @@ -124,12 +125,28 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { ) } - // because of the additional burden of having to manage reference data-plane - // configurations for HTTPRoute objects in the underlying Kong Gateway, we - // simply reconcile ALL HTTPRoute objects. This allows us to drop the backend - // data-plane config for an HTTPRoute if it somehow becomes disconnected from - // a supported Gateway and GatewayClass. - return blder.For(&gatewayapi.HTTPRoute{}). + // We enqueue only routes that are: + // - attached during creation or deletion + // - have been attached or detached to a reconciled Gateway. + // This allows us to drop the backend data-plane config for a route if + // it somehow becomes disconnected from a supported Gateway and GatewayClass. + return blder. + For(&gatewayapi.HTTPRoute{}, + builder.WithPredicates(predicate.Funcs{ + GenericFunc: func(_ event.GenericEvent) bool { + return false // we don't need to enqueue from generic + }, + CreateFunc: func(e event.CreateEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.HTTPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isOrWasRouteAttachedToReconciledGateway[*gatewayapi.HTTPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.HTTPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + }), + ). Complete(r) } @@ -137,9 +154,9 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { // HTTPRoute Controller - Event Handlers // ----------------------------------------------------------------------------- -// listReferenceGrantsForHTTPRoute is a watch predicate which finds all HTTPRoutes +// listHTTPRoutesForReferenceGrant is a watch predicate which finds all HTTPRoutes // mentioned in a From clause for a ReferenceGrant. -func (r *HTTPRouteReconciler) listReferenceGrantsForHTTPRoute(ctx context.Context, obj client.Object) []reconcile.Request { +func (r *HTTPRouteReconciler) listHTTPRoutesForReferenceGrant(ctx context.Context, obj client.Object) []reconcile.Request { grant, ok := obj.(*gatewayapi.ReferenceGrant) if !ok { r.Log.Error( @@ -155,15 +172,15 @@ func (r *HTTPRouteReconciler) listReferenceGrantsForHTTPRoute(ctx context.Contex return nil } recs := []reconcile.Request{} - for _, gateway := range httproutes.Items { + for _, httproute := range httproutes.Items { for _, from := range grant.Spec.From { - if string(from.Namespace) == gateway.Namespace && + if string(from.Namespace) == httproute.Namespace && from.Kind == ("HTTPRoute") && from.Group == ("gateway.networking.k8s.io") { recs = append(recs, reconcile.Request{ NamespacedName: k8stypes.NamespacedName{ - Namespace: gateway.Namespace, - Name: gateway.Name, + Namespace: httproute.Namespace, + Name: httproute.Name, }, }) } @@ -379,22 +396,14 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( debug(log, httproute, "Retrieving GatewayClass and Gateway for route") gateways, err := getSupportedGatewayForRoute(ctx, log, r.Client, httproute, r.GatewayNN) if err != nil { - if err.Error() == unsupportedGW { - debug(log, httproute, "Unsupported route found, processing to verify whether it was ever supported") + if errors.Is(err, ErrNoSupportedGateway) { // if there's no supported Gateway then this route could have been previously // supported by this controller. As such we ensure that no supported Gateway // references exist in the object status any longer. - statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, httproute) - if err != nil { + if _, err := ensureGatewayReferenceStatusRemoved(ctx, r.Client, log, httproute); err != nil { // some failure happened so we need to retry to avoid orphaned statuses return ctrl.Result{}, err } - if statusUpdated { - // the status did in fact needed to be updated, so no need to requeue - // as the status update will trigger a requeue. - debug(log, httproute, "Unsupported route was previously supported, status was updated") - return ctrl.Result{}, nil - } // if the route doesn't have a supported Gateway+GatewayClass associated with // it it's possible it became orphaned after becoming queued. In either case @@ -616,34 +625,6 @@ func (r *HTTPRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont return true, nil } -// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway -// implementation to prune status references to Gateways supported by this controller -// in the provided HTTPRoute object. -func (r *HTTPRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, httproute *gatewayapi.HTTPRoute) (bool, error) { - // drop all status references to supported Gateway objects - newStatuses := make([]gatewayapi.RouteParentStatus, 0) - for _, status := range httproute.Status.Parents { - if status.ControllerName != GetControllerName() { - newStatuses = append(newStatuses, status) - } - } - - // if the new list of statuses is the same length as the old - // nothing has changed and we're all done. - if len(newStatuses) == len(httproute.Status.Parents) { - return false, nil - } - - // update the object status in the API - httproute.Status.Parents = newStatuses - if err := r.Status().Update(ctx, httproute); err != nil { - return false, err - } - - // the status needed an update and it was updated successfully - return true, nil -} - // setRouteConditionResolvedRefsCondition sets a condition of type ResolvedRefs on the route status. func (r *HTTPRouteReconciler) setRouteConditionResolvedRefsCondition( ctx context.Context, diff --git a/internal/controllers/gateway/route_parent_status.go b/internal/controllers/gateway/route_parent_status.go index 37b8065fe0..90719a8fee 100644 --- a/internal/controllers/gateway/route_parent_status.go +++ b/internal/controllers/gateway/route_parent_status.go @@ -89,3 +89,35 @@ func getParentRef(parentStatus gatewayapi.RouteParentStatus) parentRef { SectionName: sectionName, } } + +func getRouteStatusParents[T gatewayapi.RouteT](route T) []gatewayapi.RouteParentStatus { + switch r := any(route).(type) { + case *gatewayapi.HTTPRoute: + return r.Status.Parents + case *gatewayapi.TCPRoute: + return r.Status.Parents + case *gatewayapi.UDPRoute: + return r.Status.Parents + case *gatewayapi.TLSRoute: + return r.Status.Parents + case *gatewayapi.GRPCRoute: + return r.Status.Parents + default: + return nil + } +} + +func setRouteStatusParents[T gatewayapi.RouteT](route T, parents []gatewayapi.RouteParentStatus) { + switch r := any(route).(type) { + case *gatewayapi.HTTPRoute: + r.Status.Parents = parents + case *gatewayapi.TCPRoute: + r.Status.Parents = parents + case *gatewayapi.UDPRoute: + r.Status.Parents = parents + case *gatewayapi.TLSRoute: + r.Status.Parents = parents + case *gatewayapi.GRPCRoute: + r.Status.Parents = parents + } +} diff --git a/internal/controllers/gateway/route_predicates.go b/internal/controllers/gateway/route_predicates.go new file mode 100644 index 0000000000..8fd02bb4f4 --- /dev/null +++ b/internal/controllers/gateway/route_predicates.go @@ -0,0 +1,108 @@ +package gateway + +import ( + "context" + "fmt" + "reflect" + + "github.com/go-logr/logr" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers" + "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" +) + +func isRouteAttachedToReconciledGateway[routeT gatewayapi.RouteT]( + cl client.Client, log logr.Logger, gatewayNN controllers.OptionalNamespacedName, obj client.Object, +) bool { + route, ok := obj.(routeT) + if !ok { + kind := obj.GetObjectKind().GroupVersionKind().Kind + log.Error( + fmt.Errorf("unexpected object type"), + "Route watch predicate received unexpected object type", + "expected", kind, "found", reflect.TypeOf(obj), + ) + return false + } + + parentRefs := getRouteParentRefs(route) + + // If the reconciler has a GatewayNN set, only HTTPRoutes attached to that Gateway are reconciled. + if gNN, ok := gatewayNN.Get(); ok { + for _, parentRef := range parentRefs { + if parentRef.Namespace != nil && string(*parentRef.Namespace) != gNN.Namespace { + continue + } + if string(parentRef.Name) != gNN.Name { + continue + } + if parentRef.Kind != nil && *parentRef.Kind != "Gateway" { + continue + } + if parentRef.Group != nil && *parentRef.Group != gatewayapi.Group(gatewayapi.GroupVersion.Group) { + continue + } + return true + } + return false + } + + // If the GatewayNN is not set, all HTTPRoutes are reconciled. + // Hence we need to check if the HTTPRoute is attached to a Gateway that is managed by this controller. + for _, parentRef := range parentRefs { + namespace := route.GetNamespace() + if parentRef.Namespace != nil { + namespace = string(*parentRef.Namespace) + } + + kind := gatewayapi.Kind("Gateway") + if parentRef.Kind != nil { + kind = *parentRef.Kind + } + + group := gatewayapi.GroupVersion.Group + if parentRef.Group != nil { + group = string(*parentRef.Group) + } + + switch { + case kind == "Gateway" && group == gatewayapi.GroupVersion.Group: + var gateway gatewayapi.Gateway + err := cl.Get(context.Background(), k8stypes.NamespacedName{Namespace: namespace, Name: string(parentRef.Name)}, &gateway) + if err != nil { + log.Error(err, "Failed to get Gateway in HTTPRoute watch") + return false + } + + var gatewayClass gatewayapi.GatewayClass + err = cl.Get(context.Background(), k8stypes.NamespacedName{Name: string(gateway.Spec.GatewayClassName)}, &gatewayClass) + if err != nil { + log.Error(err, "Failed to get GatewayClass in HTTPRoute watch") + return false + } + + if isGatewayClassControlled(&gatewayClass) { + return true + } + default: + log.Error( + fmt.Errorf("unsupported parentRef kind %s and group %s", kind, group), + "Got an unexpected kind and group when checking route's parentRefs", + ) + return false + } + } + + return false +} + +func isOrWasRouteAttachedToReconciledGateway[routeT gatewayapi.RouteT]( + cl client.Client, log logr.Logger, gatewayNN controllers.OptionalNamespacedName, e event.UpdateEvent, +) bool { + oldObj, newObj := e.ObjectOld, e.ObjectNew + return isRouteAttachedToReconciledGateway[routeT](cl, log, gatewayNN, oldObj) || + isRouteAttachedToReconciledGateway[routeT](cl, log, gatewayNN, newObj) +} diff --git a/internal/controllers/gateway/route_utils.go b/internal/controllers/gateway/route_utils.go index 41677cc1a3..df6a921aea 100644 --- a/internal/controllers/gateway/route_utils.go +++ b/internal/controllers/gateway/route_utils.go @@ -26,10 +26,6 @@ import ( // Route Utilities // ----------------------------------------------------------------------------- -const ( - unsupportedGW = "no supported Gateway found for route" -) - const ( ConditionTypeProgrammed = "Programmed" ConditionReasonProgrammedUnknown gatewayapi.RouteConditionReason = "Unknown" @@ -37,7 +33,10 @@ const ( ConditionReasonTranslationError gatewayapi.RouteConditionReason = "TranslationError" ) -var ErrNoMatchingListenerHostname = fmt.Errorf("no matching hostnames in listener") +var ( + ErrNoMatchingListenerHostname = fmt.Errorf("no matching hostnames in listener") + ErrNoSupportedGateway = fmt.Errorf("no supported gateway found for route") +) // supportedGatewayWithCondition is a struct that wraps a gateway and some further info // such as the condition Status condition Accepted of the gateway and the listenerName. @@ -351,7 +350,7 @@ func getSupportedGatewayForRoute[T gatewayapi.RouteT](ctx context.Context, logge } if len(gateways) == 0 { - return nil, fmt.Errorf(unsupportedGW) + return nil, ErrNoSupportedGateway } return gateways, nil @@ -985,3 +984,65 @@ func isRouteAcceptedByListener[T gatewayapi.RouteT](ctx context.Context, return true, nil } + +// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway +// implementation to prune status references to Gateways supported by this controller +// in the provided route. +func ensureGatewayReferenceStatusRemoved[routeT gatewayapi.RouteT]( + ctx context.Context, cl client.Client, log logr.Logger, route routeT, +) (bool, error) { + debug(log, route, "Unsupported route found, processing to verify whether it was ever supported") + kind := route.GetObjectKind().GroupVersionKind().Kind + parents := getRouteStatusParents(route) + + // Drop all status references to supported Gateway objects. + newStatuses := make([]gatewayapi.RouteParentStatus, 0) + for _, status := range parents { + if status.ControllerName != GetControllerName() { + newStatuses = append(newStatuses, status) + } else { + parentRefNN := string(status.ParentRef.Name) + if status.ParentRef.Namespace != nil { + parentRefNN = fmt.Sprintf("%s/%s", *status.ParentRef.Namespace, parentRefNN) + } + debug(log, route, "Removing parentRef from route status", "parentRef", parentRefNN, "kind", kind) + } + } + + // If the new list of statuses is the same length as the old + // nothing has changed and we're all done. + if len(newStatuses) == len(parents) { + return false, nil + } + + // If the route doesn't have a supported Gateway+GatewayClass associated with + // it it's possible it became orphaned after becoming queued. In either case + // ensure that it's removed from the proxy cache to avoid orphaned data-plane + // configurations. + debug(log, route, "Ensuring that dataplane is updated to remove unsupported route (if applicable)") + setRouteStatusParents(route, newStatuses) + if err := cl.Status().Update(ctx, route); err != nil { + return false, fmt.Errorf("failed to remove Gateway parentRef from %s status: %w", kind, err) + } + + debug(log, route, "Unsupported route was previously supported, status was updated") + // The status needed to be updated and it was updated successfully. + return true, nil +} + +func getRouteParentRefs[T gatewayapi.RouteT](route T) []gatewayapi.ParentReference { + switch r := any(route).(type) { + case *gatewayapi.HTTPRoute: + return r.Spec.ParentRefs + case *gatewayapi.TCPRoute: + return r.Spec.ParentRefs + case *gatewayapi.UDPRoute: + return r.Spec.ParentRefs + case *gatewayapi.TLSRoute: + return r.Spec.ParentRefs + case *gatewayapi.GRPCRoute: + return r.Spec.ParentRefs + default: + return nil + } +} diff --git a/internal/controllers/gateway/route_utils_test.go b/internal/controllers/gateway/route_utils_test.go index c839c72470..e61843e4e0 100644 --- a/internal/controllers/gateway/route_utils_test.go +++ b/internal/controllers/gateway/route_utils_test.go @@ -1432,7 +1432,7 @@ func TestGetSupportedGatewayForRoute(t *testing.T) { namespace, }, expected: []expected{}, - expectedErr: fmt.Errorf("no supported Gateway found for route"), + expectedErr: ErrNoSupportedGateway, }, } diff --git a/internal/controllers/gateway/tcproute_controller.go b/internal/controllers/gateway/tcproute_controller.go index 1482b10ef9..03ae617628 100644 --- a/internal/controllers/gateway/tcproute_controller.go +++ b/internal/controllers/gateway/tcproute_controller.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "reflect" "time" @@ -95,12 +96,28 @@ func (r *TCPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { ) } - // because of the additional burden of having to manage reference data-plane - // configurations for TCPRoute objects in the underlying Kong Gateway, we - // simply reconcile ALL TCPRoute objects. This allows us to drop the backend - // data-plane config for an TCPRoute if it somehow becomes disconnected from - // a supported Gateway and GatewayClass. - return blder.For(&gatewayapi.TCPRoute{}). + // We enqueue only routes that are: + // - attached during creation or deletion + // - have been attached or detached to a reconciled Gateway. + // This allows us to drop the backend data-plane config for a route if + // it somehow becomes disconnected from a supported Gateway and GatewayClass. + return blder. + For(&gatewayapi.TCPRoute{}, + builder.WithPredicates(predicate.Funcs{ + GenericFunc: func(_ event.GenericEvent) bool { + return false // we don't need to enqueue from generic + }, + CreateFunc: func(e event.CreateEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.TCPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isOrWasRouteAttachedToReconciledGateway[*gatewayapi.TCPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.TCPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + }), + ). Complete(r) } @@ -301,22 +318,14 @@ func (r *TCPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c debug(log, tcproute, "Retrieving GatewayClass and Gateway for route") gateways, err := getSupportedGatewayForRoute(ctx, log, r.Client, tcproute, r.GatewayNN) if err != nil { - if err.Error() == unsupportedGW { - debug(log, tcproute, "Unsupported route found, processing to verify whether it was ever supported") + if errors.Is(err, ErrNoSupportedGateway) { // if there's no supported Gateway then this route could have been previously // supported by this controller. As such we ensure that no supported Gateway // references exist in the object status any longer. - statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, tcproute) - if err != nil { + if _, err := ensureGatewayReferenceStatusRemoved(ctx, r.Client, log, tcproute); err != nil { // some failure happened so we need to retry to avoid orphaned statuses return ctrl.Result{}, err } - if statusUpdated { - // the status did in fact needed to be updated, so no need to requeue - // as the status update will trigger a requeue. - debug(log, tcproute, "Unsupported route was previously supported, status was updated") - return ctrl.Result{}, nil - } // if the route doesn't have a supported Gateway+GatewayClass associated with // it it's possible it became orphaned after becoming queued. In either case @@ -527,31 +536,3 @@ func (r *TCPRouteReconciler) ensureGatewayReferenceStatusAdded( // the status needed an update and it was updated successfully return true, nil } - -// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway -// implementation to prune status references to Gateways supported by this controller -// in the provided TCPRoute object. -func (r *TCPRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, tcproute *gatewayapi.TCPRoute) (bool, error) { - // drop all status references to supported Gateway objects - newStatuses := make([]gatewayapi.RouteParentStatus, 0) - for _, status := range tcproute.Status.Parents { - if status.ControllerName != GetControllerName() { - newStatuses = append(newStatuses, status) - } - } - - // if the new list of statuses is the same length as the old - // nothing has changed and we're all done. - if len(newStatuses) == len(tcproute.Status.Parents) { - return false, nil - } - - // update the object status in the API - tcproute.Status.Parents = newStatuses - if err := r.Status().Update(ctx, tcproute); err != nil { - return false, err - } - - // the status needed an update and it was updated successfully - return true, nil -} diff --git a/internal/controllers/gateway/tlsroute_controller.go b/internal/controllers/gateway/tlsroute_controller.go index ed7cd93d99..6b64026e62 100644 --- a/internal/controllers/gateway/tlsroute_controller.go +++ b/internal/controllers/gateway/tlsroute_controller.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "reflect" "time" @@ -90,12 +91,28 @@ func (r *TLSRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { ) } - // because of the additional burden of having to manage reference data-plane - // configurations for TLSRoute objects in the underlying Kong Gateway, we - // simply reconcile ALL TLSRoute objects. This allows us to drop the backend - // data-plane config for an TLSRoute if it somehow becomes disconnected from - // a supported Gateway and GatewayClass. - return blder.For(&gatewayapi.TLSRoute{}). + // We enqueue only routes that are: + // - attached during creation or deletion + // - have been attached or detached to a reconciled Gateway. + // This allows us to drop the backend data-plane config for a route if + // it somehow becomes disconnected from a supported Gateway and GatewayClass. + return blder. + For(&gatewayapi.TLSRoute{}, + builder.WithPredicates(predicate.Funcs{ + GenericFunc: func(_ event.GenericEvent) bool { + return false // we don't need to enqueue from generic + }, + CreateFunc: func(e event.CreateEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.TLSRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isOrWasRouteAttachedToReconciledGateway[*gatewayapi.TLSRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.TLSRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + }), + ). Complete(r) } @@ -296,22 +313,14 @@ func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c debug(log, tlsroute, "Retrieving GatewayClass and Gateway for route") gateways, err := getSupportedGatewayForRoute(ctx, log, r.Client, tlsroute, r.GatewayNN) if err != nil { - if err.Error() == unsupportedGW { - debug(log, tlsroute, "Unsupported route found, processing to verify whether it was ever supported") + if errors.Is(err, ErrNoSupportedGateway) { // if there's no supported Gateway then this route could have been previously // supported by this controller. As such we ensure that no supported Gateway // references exist in the object status any longer. - statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, tlsroute) - if err != nil { + if _, err := ensureGatewayReferenceStatusRemoved(ctx, r.Client, log, tlsroute); err != nil { // some failure happened so we need to retry to avoid orphaned statuses return ctrl.Result{}, err } - if statusUpdated { - // the status did in fact needed to be updated, so no need to requeue - // as the status update will trigger a requeue. - debug(log, tlsroute, "Unsupported route was previously supported, status was updated") - return ctrl.Result{}, nil - } // if the route doesn't have a supported Gateway+GatewayClass associated with // it it's possible it became orphaned after becoming queued. In either case @@ -513,31 +522,3 @@ func (r *TLSRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Conte // the status needed an update and it was updated successfully return true, nil } - -// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway -// implementation to prune status references to Gateways supported by this controller -// in the provided TLSRoute object. -func (r *TLSRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, tlsroute *gatewayapi.TLSRoute) (bool, error) { - // drop all status references to supported Gateway objects - newStatuses := make([]gatewayapi.RouteParentStatus, 0) - for _, status := range tlsroute.Status.Parents { - if status.ControllerName != GetControllerName() { - newStatuses = append(newStatuses, status) - } - } - - // if the new list of statuses is the same length as the old - // nothing has changed and we're all done. - if len(newStatuses) == len(tlsroute.Status.Parents) { - return false, nil - } - - // update the object status in the API - tlsroute.Status.Parents = newStatuses - if err := r.Status().Update(ctx, tlsroute); err != nil { - return false, err - } - - // the status needed an update and it was updated successfully - return true, nil -} diff --git a/internal/controllers/gateway/udproute_controller.go b/internal/controllers/gateway/udproute_controller.go index f91396000b..09e52c3e4a 100644 --- a/internal/controllers/gateway/udproute_controller.go +++ b/internal/controllers/gateway/udproute_controller.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "reflect" "time" @@ -94,12 +95,28 @@ func (r *UDPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { ) } - // because of the additional burden of having to manage reference data-plane - // configurations for UDPRoute objects in the underlying Kong Gateway, we - // simply reconcile ALL UDPRoute objects. This allows us to drop the backend - // data-plane config for an UDPRoute if it somehow becomes disconnected from - // a supported Gateway and GatewayClass. - return blder.For(&gatewayapi.UDPRoute{}). + // We enqueue only routes that are: + // - attached during creation or deletion + // - have been attached or detached to a reconciled Gateway. + // This allows us to drop the backend data-plane config for a route if + // it somehow becomes disconnected from a supported Gateway and GatewayClass. + return blder. + For(&gatewayapi.UDPRoute{}, + builder.WithPredicates(predicate.Funcs{ + GenericFunc: func(_ event.GenericEvent) bool { + return false // We don't need to enqueue from generic. + }, + CreateFunc: func(e event.CreateEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.UDPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isOrWasRouteAttachedToReconciledGateway[*gatewayapi.UDPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return isRouteAttachedToReconciledGateway[*gatewayapi.UDPRoute](r.Client, mgr.GetLogger(), r.GatewayNN, e.Object) + }, + }), + ). Complete(r) } @@ -300,22 +317,15 @@ func (r *UDPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c debug(log, udproute, "Retrieving GatewayClass and Gateway for route") gateways, err := getSupportedGatewayForRoute(ctx, log, r.Client, udproute, r.GatewayNN) if err != nil { - if err.Error() == unsupportedGW { - debug(log, udproute, "Unsupported route found, processing to verify whether it was ever supported") + if errors.Is(err, ErrNoSupportedGateway) { // if there's no supported Gateway then this route could have been previously // supported by this controller. As such we ensure that no supported Gateway // references exist in the object status any longer. - statusUpdated, err := r.ensureGatewayReferenceStatusRemoved(ctx, udproute) + _, err := ensureGatewayReferenceStatusRemoved(ctx, r.Client, log, udproute) if err != nil { // some failure happened so we need to retry to avoid orphaned statuses return ctrl.Result{}, err } - if statusUpdated { - // the status did in fact needed to be updated, so no need to requeue - // as the status update will trigger a requeue. - debug(log, udproute, "Unsupported route was previously supported, status was updated") - return ctrl.Result{}, nil - } // if the route doesn't have a supported Gateway+GatewayClass associated with // it it's possible it became orphaned after becoming queued. In either case @@ -517,31 +527,3 @@ func (r *UDPRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Conte // the status needed an update and it was updated successfully return true, nil } - -// ensureGatewayReferenceStatusRemoved uses the ControllerName provided by the Gateway -// implementation to prune status references to Gateways supported by this controller -// in the provided UDPRoute object. -func (r *UDPRouteReconciler) ensureGatewayReferenceStatusRemoved(ctx context.Context, udproute *gatewayapi.UDPRoute) (bool, error) { - // drop all status references to supported Gateway objects - newStatuses := make([]gatewayapi.RouteParentStatus, 0) - for _, status := range udproute.Status.Parents { - if status.ControllerName != GetControllerName() { - newStatuses = append(newStatuses, status) - } - } - - // if the new list of statuses is the same length as the old - // nothing has changed and we're all done. - if len(newStatuses) == len(udproute.Status.Parents) { - return false, nil - } - - // update the object status in the API - udproute.Status.Parents = newStatuses - if err := r.Status().Update(ctx, udproute); err != nil { - return false, err - } - - // the status needed an update and it was updated successfully - return true, nil -} diff --git a/internal/gatewayapi/aliases.go b/internal/gatewayapi/aliases.go index a38cd18531..d02c553595 100644 --- a/internal/gatewayapi/aliases.go +++ b/internal/gatewayapi/aliases.go @@ -6,7 +6,10 @@ import ( gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) -var InstallV1 = gatewayv1.Install +var ( + InstallV1 = gatewayv1.Install + GroupVersion = gatewayv1.GroupVersion +) // This file contains aliases for types and consts from the Gateway API. Its purpose is to allow easy migration from // one version of the Gateway API to another with minimal changes to the codebase. @@ -20,6 +23,7 @@ type ( Gateway = gatewayv1.Gateway GatewayAddress = gatewayv1.GatewayAddress GatewayClass = gatewayv1.GatewayClass + GatewayClassList = gatewayv1.GatewayClassList GatewayClassSpec = gatewayv1.GatewayClassSpec GatewayClassStatus = gatewayv1.GatewayClassStatus GatewayController = gatewayv1.GatewayController diff --git a/test/envtest/httproute_controller_test.go b/test/envtest/httproute_controller_test.go index a8fb519c84..2a26e9bd76 100644 --- a/test/envtest/httproute_controller_test.go +++ b/test/envtest/httproute_controller_test.go @@ -291,7 +291,7 @@ func TestHTTPRouteReconciler_RemovesOutdatedParentStatuses(t *testing.T) { ControllerName: gateway.GetControllerName(), }, ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), + GenerateName: "kong-gwclass-", Annotations: map[string]string{ "konghq.com/gatewayclass-unmanaged": "placeholder", }, @@ -299,29 +299,54 @@ func TestHTTPRouteReconciler_RemovesOutdatedParentStatuses(t *testing.T) { } require.NoError(t, client.Create(ctx, &gwc)) t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) + gwcNonKong := gatewayapi.GatewayClass{ + Spec: gatewayapi.GatewayClassSpec{ + ControllerName: "acme.com/dummy-controller", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "non-kong-gwclass-", + }, + } + require.NoError(t, client.Create(ctx, &gwcNonKong)) + t.Cleanup(func() { _ = client.Delete(ctx, &gwcNonKong) }) gw := gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + GenerateName: "gw-kong-", + }, Spec: gatewayapi.GatewaySpec{ GatewayClassName: gatewayapi.ObjectName(gwc.Name), Listeners: []gatewayapi.Listener{ { - Name: gatewayapi.SectionName("http"), - Port: gatewayapi.PortNumber(80), - Protocol: gatewayapi.HTTPProtocolType, - AllowedRoutes: &gatewayapi.AllowedRoutes{ - Namespaces: &gatewayapi.RouteNamespaces{ - From: lo.ToPtr(gatewayapi.NamespacesFromAll), - }, - }, + Name: gatewayapi.SectionName("http"), + Port: gatewayapi.PortNumber(80), + Protocol: gatewayapi.HTTPProtocolType, + AllowedRoutes: builder.NewAllowedRoutesFromAllNamespaces(), }, }, }, + } + require.NoError(t, client.Create(ctx, &gw)) + + gwNonKong := gatewayapi.Gateway{ ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: uuid.NewString(), + GenerateName: "gw-nonkong-", + Namespace: ns.Name, + }, + Spec: gatewayapi.GatewaySpec{ + GatewayClassName: gatewayapi.ObjectName(gwcNonKong.Name), + Listeners: []gatewayapi.Listener{ + { + Name: gatewayapi.SectionName("http"), + Port: gatewayapi.PortNumber(80), + Protocol: gatewayapi.HTTPProtocolType, + AllowedRoutes: builder.NewAllowedRoutesFromAllNamespaces(), + }, + }, }, } - require.NoError(t, client.Create(ctx, &gw)) + require.NoError(t, client.Create(ctx, &gwNonKong)) route := gatewayapi.HTTPRoute{ TypeMeta: metav1.TypeMeta{ @@ -329,8 +354,8 @@ func TestHTTPRouteReconciler_RemovesOutdatedParentStatuses(t *testing.T) { APIVersion: "v1beta1", }, ObjectMeta: metav1.ObjectMeta{ - Namespace: nsRoute.Name, - Name: uuid.NewString(), + Namespace: nsRoute.Name, + GenerateName: "httproute-kong-", }, Spec: gatewayapi.HTTPRouteSpec{ CommonRouteSpec: gatewayapi.CommonRouteSpec{ @@ -345,14 +370,13 @@ func TestHTTPRouteReconciler_RemovesOutdatedParentStatuses(t *testing.T) { }, } require.NoError(t, client.Create(ctx, &route)) - // Status has to be updated separately. route.Status = gatewayapi.HTTPRouteStatus{ RouteStatus: gatewayapi.RouteStatus{ Parents: []gatewayapi.RouteParentStatus{ { ParentRef: gatewayapi.ParentReference{ - Name: "other-gw-name", + Name: gatewayapi.ObjectName(gwNonKong.Name), }, ControllerName: gateway.GetControllerName(), }, @@ -361,22 +385,99 @@ func TestHTTPRouteReconciler_RemovesOutdatedParentStatuses(t *testing.T) { } require.NoError(t, client.Status().Update(ctx, &route)) - require.Eventually(t, func() bool { - err := client.Get(ctx, k8stypes.NamespacedName{Name: route.Name, Namespace: route.Namespace}, &route) - if err != nil { - t.Logf("failed to get HTTPRoute %s/%s: %v", route.Namespace, route.Name, err) - return false - } + routeNonKong := gatewayapi.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsRoute.Name, + GenerateName: "httproute-nonkong-", + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Name: gatewayapi.ObjectName(gwNonKong.Name), + Namespace: lo.ToPtr(gatewayapi.Namespace(ns.Name)), + }}, + }, + Rules: []gatewayapi.HTTPRouteRule{{ + BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), + }}, + }, + } + require.NoError(t, client.Create(ctx, &routeNonKong)) + // Status has to be updated separately. + routeNonKong.Status = gatewayapi.HTTPRouteStatus{ + RouteStatus: gatewayapi.RouteStatus{ + Parents: []gatewayapi.RouteParentStatus{ + { + ParentRef: gatewayapi.ParentReference{ + Name: gatewayapi.ObjectName(gwNonKong.Name), + }, + ControllerName: gwcNonKong.Spec.ControllerName, + }, + }, + }, + } + require.NoError(t, client.Status().Update(ctx, &routeNonKong)) + + t.Run("routes attached to Gateways that are reconciled by KIC should have other Gateway refs cleared from status", func(t *testing.T) { + require.Eventually(t, func() bool { + if err := client.Get(ctx, ctrlclient.ObjectKeyFromObject(&route), &route); err != nil { + t.Logf("failed to get HTTPRoute %s: %v", ctrlclient.ObjectKeyFromObject(&route), err) + return false + } + + if staleStatusFound := lo.ContainsBy(route.Status.Parents, func(ps gatewayapi.RouteParentStatus) bool { + return string(ps.ParentRef.Name) == gwNonKong.Name + }); staleStatusFound { + t.Logf("found stale status for parent Gateway %s that does not belong to KIC GatewayClass", gwNonKong.Name) + return false + } + + return true + }, waitDuration, tickDuration, "expected stale status to be removed from HTTPRoute") + }) + + t.Run("routes that were attached to Gateways that are reconciled by KIC and now become detached should have KIC Gateway refs cleared from status", func(t *testing.T) { + require.Eventually(t, func() bool { + if err := client.Get(ctx, ctrlclient.ObjectKeyFromObject(&route), &route); err != nil { + t.Logf("failed to get HTTPRoute %s: %v", ctrlclient.ObjectKeyFromObject(&route), err) + return false + } + route.Spec.ParentRefs = nil + if err := client.Status().Update(ctx, &route); err != nil { + t.Logf("failed to update HTTPRoute %s: %v", ctrlclient.ObjectKeyFromObject(&route), err) + return false + } + + if err := client.Get(ctx, ctrlclient.ObjectKeyFromObject(&route), &route); err != nil { + t.Logf("failed to get HTTPRoute %s: %v", ctrlclient.ObjectKeyFromObject(&route), err) + return false + } + + return len(route.Status.Parents) == 0 + }, waitDuration, tickDuration, "expected stale KIC Gateway parentRef to be removed from HTTPRoute status") + }) + + t.Run("routes attached to Gateways that are not reconciled by KIC should not have other Gateway refs cleared from status", func(t *testing.T) { + require.Never(t, func() bool { + if err := client.Get(ctx, ctrlclient.ObjectKeyFromObject(&routeNonKong), &routeNonKong); err != nil { + t.Logf("failed to get HTTPRoute %s: %v", ctrlclient.ObjectKeyFromObject(&routeNonKong), err) + return true + } + + if staleStatusFound := lo.ContainsBy(routeNonKong.Status.Parents, func(ps gatewayapi.RouteParentStatus) bool { + return string(ps.ParentRef.Name) == gwNonKong.Name + }); !staleStatusFound { + t.Logf("status for parent %s not found, it should not have been cleared", gwNonKong.Name) + return true + } - if staleStatusFound := lo.ContainsBy(route.Status.Parents, func(ps gatewayapi.RouteParentStatus) bool { - return ps.ParentRef.Name == "other-gw-name" - }); staleStatusFound { - t.Log("found stale status for parent other-gw-name") return false - } - - return true - }, waitDuration, tickDuration, "expected stale status to be removed from HTTPRoute") + }, waitDuration, tickDuration, "expected status to not be removed from HTTPRoute") + }) } func printHTTPRoutesConditions(ctx context.Context, client ctrlclient.Client, nn k8stypes.NamespacedName) string { diff --git a/test/internal/helpers/gatewayapi.go b/test/internal/helpers/gatewayapi.go index cfa42fea88..e0d2780758 100644 --- a/test/internal/helpers/gatewayapi.go +++ b/test/internal/helpers/gatewayapi.go @@ -119,7 +119,7 @@ func gatewayLinkStatusMatches( groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) - t.Logf("error getting grpc route: %v", err) + t.Logf("error getting grpc route: %v", gerr) } else { if err == nil { return newRouteParentsStatus(route.Status.Parents). From cf2efdaa3506bd43dadbeadca5af896d668c3d6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 16:57:24 +0200 Subject: [PATCH 08/48] chore(deps): bump sigs.k8s.io/controller-runtime from 0.18.2 to 0.18.3 (#6094) Bumps [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) from 0.18.2 to 0.18.3. - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Changelog](https://github.com/kubernetes-sigs/controller-runtime/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.18.2...v0.18.3) --- updated-dependencies: - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bfcbc45000..efe1693145 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( k8s.io/apimachinery v0.30.1 k8s.io/client-go v0.30.1 k8s.io/component-base v0.30.1 - sigs.k8s.io/controller-runtime v0.18.2 + sigs.k8s.io/controller-runtime v0.18.3 sigs.k8s.io/gateway-api v1.1.0 sigs.k8s.io/kustomize/api v0.17.2 sigs.k8s.io/kustomize/kyaml v0.17.1 diff --git a/go.sum b/go.sum index ac6afa30df..9a2e72550e 100644 --- a/go.sum +++ b/go.sum @@ -657,8 +657,8 @@ k8s.io/kubectl v0.30.1 h1:sHFIRI3oP0FFZmBAVEE8ErjnTyXDPkBcvO88mH9RjuY= k8s.io/kubectl v0.30.1/go.mod h1:7j+L0Cc38RYEcx+WH3y44jRBe1Q1jxdGPKkX0h4iDq0= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= -sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= +sigs.k8s.io/controller-runtime v0.18.3 h1:B5Wmmo8WMWK7izei+2LlXLVDGzMwAHBNLX68lwtlSR4= +sigs.k8s.io/controller-runtime v0.18.3/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/e2e-framework v0.3.1-0.20231113122213-262cac32d35e h1:lJqSZb2bAyfkPpBhUbzXsoAHKJn+3/KzBpvktR1wlMQ= sigs.k8s.io/e2e-framework v0.3.1-0.20231113122213-262cac32d35e/go.mod h1:VIozg+of0zhkVGZcCWvKqH7vn7GTDDAa3h+w1OfH7Co= sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= From fd03b86eab52a9800c05a3978595c831ab7d02c4 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Tue, 28 May 2024 10:31:56 +0200 Subject: [PATCH 09/48] feat: use TakeSnapshotIfChanged to avoid taking redundant snapshot (#6092) --- internal/dataplane/fallback/fallback.go | 6 +- internal/dataplane/fallback/fallback_test.go | 12 +-- internal/dataplane/kong_client.go | 20 +++-- internal/dataplane/kong_client_test.go | 78 ++++++++++++++++++-- internal/store/cache_stores_snapshot.go | 8 +- 5 files changed, 99 insertions(+), 25 deletions(-) diff --git a/internal/dataplane/fallback/fallback.go b/internal/dataplane/fallback/fallback.go index c3643aacff..add1eea399 100644 --- a/internal/dataplane/fallback/fallback.go +++ b/internal/dataplane/fallback/fallback.go @@ -27,8 +27,8 @@ func NewGenerator(cacheGraphProvider CacheGraphProvider, logger logr.Logger) *Ge } } -// GenerateExcludingAffected generates a new cache snapshot that excludes all objects that depend on the broken objects. -func (g *Generator) GenerateExcludingAffected( +// GenerateExcludingBrokenObjects generates a new cache snapshot that excludes all objects that depend on the broken objects. +func (g *Generator) GenerateExcludingBrokenObjects( cache store.CacheStores, brokenObjects []ObjectHash, ) (store.CacheStores, error) { @@ -37,7 +37,7 @@ func (g *Generator) GenerateExcludingAffected( return store.CacheStores{}, fmt.Errorf("failed to build cache graph: %w", err) } - fallbackCache, err := cache.TakeSnapshot() + fallbackCache, _, err := cache.TakeSnapshotIfChanged(store.SnapshotHashEmpty) if err != nil { return store.CacheStores{}, fmt.Errorf("failed to take cache snapshot: %w", err) } diff --git a/internal/dataplane/fallback/fallback_test.go b/internal/dataplane/fallback/fallback_test.go index 59201e9c3e..cce0a9eb36 100644 --- a/internal/dataplane/fallback/fallback_test.go +++ b/internal/dataplane/fallback/fallback_test.go @@ -22,7 +22,7 @@ func (m *mockGraphProvider) CacheToGraph(s store.CacheStores) (*fallback.ConfigG return m.graph, nil } -func TestGenerator_GenerateExcludingAffected(t *testing.T) { +func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { // We have to use real-world object types here as we're testing integration with store.CacheStores. ingressClass := testIngressClass(t, "ingressClass") service := testService(t, "service") @@ -57,7 +57,7 @@ func TestGenerator_GenerateExcludingAffected(t *testing.T) { g := fallback.NewGenerator(graphProvider, logr.Discard()) t.Run("ingressClass is broken", func(t *testing.T) { - fallbackCache, err := g.GenerateExcludingAffected(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass)}) + fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass)}) require.NoError(t, err) require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) @@ -68,7 +68,7 @@ func TestGenerator_GenerateExcludingAffected(t *testing.T) { }) t.Run("service is broken", func(t *testing.T) { - fallbackCache, err := g.GenerateExcludingAffected(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(service)}) + fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(service)}) require.NoError(t, err) require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) @@ -79,7 +79,7 @@ func TestGenerator_GenerateExcludingAffected(t *testing.T) { }) t.Run("serviceFacade is broken", func(t *testing.T) { - fallbackCache, err := g.GenerateExcludingAffected(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(serviceFacade)}) + fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(serviceFacade)}) require.NoError(t, err) require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) @@ -90,7 +90,7 @@ func TestGenerator_GenerateExcludingAffected(t *testing.T) { }) t.Run("plugin is broken", func(t *testing.T) { - fallbackCache, err := g.GenerateExcludingAffected(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(plugin)}) + fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(plugin)}) require.NoError(t, err) require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) @@ -101,7 +101,7 @@ func TestGenerator_GenerateExcludingAffected(t *testing.T) { }) t.Run("multiple objects are broken", func(t *testing.T) { - fallbackCache, err := g.GenerateExcludingAffected(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass), fallback.GetObjectHash(service)}) + fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass), fallback.GetObjectHash(service)}) require.NoError(t, err) require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 91435bf138..527a9edd84 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -62,7 +62,7 @@ type KongConfigBuilder interface { // FallbackConfigGenerator generates a fallback configuration based on a cache snapshot and a set of broken objects. type FallbackConfigGenerator interface { - GenerateExcludingAffected(store.CacheStores, []fallback.ObjectHash) (store.CacheStores, error) + GenerateExcludingBrokenObjects(store.CacheStores, []fallback.ObjectHash) (store.CacheStores, error) } // KongClient is a threadsafe high level API client for the Kong data-plane(s) @@ -151,6 +151,10 @@ type KongClient struct { // fallbackConfigGenerator is used to generate a fallback configuration in case of sync failures. fallbackConfigGenerator FallbackConfigGenerator + + // lastProcessedSnapshotHash stores the hash of the last processed Kubernetes objects cache snapshot. It's used to determine configuration + // changes. Please note it is always empty when the `FallbackConfiguration` feature gate is turned off. + lastProcessedSnapshotHash store.SnapshotHash } // NewKongClient provides a new KongClient object after connecting to the @@ -413,13 +417,19 @@ func (c *KongClient) Update(ctx context.Context) error { // based on the cache contents, we need to ensure it is not modified during the process. var cacheSnapshot store.CacheStores if c.kongConfig.FallbackConfiguration { - // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6080 - // Use TakeSnapshotIfChanged to avoid taking a snapshot if the cache hasn't changed. + var newSnapshotHash store.SnapshotHash var err error - cacheSnapshot, err = c.cache.TakeSnapshot() + cacheSnapshot, newSnapshotHash, err = c.cache.TakeSnapshotIfChanged(c.lastProcessedSnapshotHash) if err != nil { return fmt.Errorf("failed to take snapshot of cache: %w", err) } + // Empty snapshot hash means that the cache hasn't changed since the last snapshot was taken. That optimization can be used + // in main code path to avoid unnecessary processing. TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6095 + if newSnapshotHash == store.SnapshotHashEmpty { + c.logger.V(util.DebugLevel).Info("No configuration change; pushing config to gateway is not necessary, skipping") + return nil + } + c.lastProcessedSnapshotHash = newSnapshotHash c.kongConfigBuilder.UpdateCache(cacheSnapshot) } @@ -518,7 +528,7 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( if err != nil { return fmt.Errorf("failed to extract broken objects from update error: %w", err) } - fallbackCache, err := c.fallbackConfigGenerator.GenerateExcludingAffected( + fallbackCache, err := c.fallbackConfigGenerator.GenerateExcludingBrokenObjects( cacheSnapshot, brokenObjects, ) diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index 27858aa95a..a5a88912d1 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -291,20 +291,20 @@ func (m mockConfigurationChangeDetector) HasConfigurationChanged( // mockKongLastValidConfigFetcher is a mock implementation of FallbackConfigGenerator interface. type mockFallbackConfigGenerator struct { - generateExcludingAffectedCalledWith lo.Tuple2[store.CacheStores, []fallback.ObjectHash] - generateExcludingAffectedResult store.CacheStores + GenerateExcludingBrokenObjectsCalledWith lo.Tuple2[store.CacheStores, []fallback.ObjectHash] + GenerateExcludingBrokenObjectsResult store.CacheStores } func newMockFallbackConfigGenerator() *mockFallbackConfigGenerator { return &mockFallbackConfigGenerator{} } -func (m *mockFallbackConfigGenerator) GenerateExcludingAffected( +func (m *mockFallbackConfigGenerator) GenerateExcludingBrokenObjects( stores store.CacheStores, hashes []fallback.ObjectHash, ) (store.CacheStores, error) { - m.generateExcludingAffectedCalledWith = lo.T2(stores, hashes) - return m.generateExcludingAffectedResult, nil + m.GenerateExcludingBrokenObjectsCalledWith = lo.T2(stores, hashes) + return m.GenerateExcludingBrokenObjectsResult, nil } func TestKongClientUpdate_AllExpectedClientsAreCalledAndErrorIsPropagated(t *testing.T) { @@ -1026,7 +1026,7 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { t.Log("Setting the fallback config generator to return a snapshot excluding the broken consumer") fallbackCacheStoresToBeReturned := cacheStoresFromObjs(t, validConsumer) - fallbackConfigGenerator.generateExcludingAffectedResult = fallbackCacheStoresToBeReturned + fallbackConfigGenerator.GenerateExcludingBrokenObjectsResult = fallbackCacheStoresToBeReturned t.Log("Calling KongClient.Update") err = kongClient.Update(ctx) @@ -1044,8 +1044,8 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { require.True(t, hasConsumer, "expected consumer to be in the first cache snapshot") t.Log("Verifying that the fallback config generator was called with the first cache snapshot and the broken object hash") - expectedGenerateExcludingAffectedArgs := lo.T2(firstCacheUpdate, []fallback.ObjectHash{fallback.GetObjectHash(brokenConsumer)}) - require.Equal(t, expectedGenerateExcludingAffectedArgs, fallbackConfigGenerator.generateExcludingAffectedCalledWith, + expectedGenerateExcludingBrokenObjectsArgs := lo.T2(firstCacheUpdate, []fallback.ObjectHash{fallback.GetObjectHash(brokenConsumer)}) + require.Equal(t, expectedGenerateExcludingBrokenObjectsArgs, fallbackConfigGenerator.GenerateExcludingBrokenObjectsCalledWith, "expected fallback config generator to be called with the first cache snapshot and the broken object hash") t.Log("Verifying that the second config builder cache update contains the fallback snapshot") @@ -1068,6 +1068,68 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { require.Equal(t, validConsumer.Username, *lastValidConfig.Consumers[0].Username) } +func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing.T) { + ctx := context.Background() + gwClient := mustSampleGatewayClient(t) + konnectClient := mustSampleKonnectClient(t) + clientsProvider := mockGatewayClientsProvider{ + gatewayClients: []*adminapi.Client{gwClient}, + konnectClient: konnectClient, + } + updateStrategyResolver := newMockUpdateStrategyResolver(t) + configChangeDetector := mockConfigurationChangeDetector{hasConfigurationChanged: true} + configBuilder := newMockKongConfigBuilder() + lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} + fallbackConfigGenerator := newMockFallbackConfigGenerator() + + // We'll use KongConsumer as an example of an object, but it could be any supported type + // for the purpose of this test as the fallback config generator is mocked anyway. + someConsumer := func(name string) *kongv1.KongConsumer { + return &kongv1.KongConsumer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "namespace", + Annotations: map[string]string{ + annotations.IngressClassKey: annotations.DefaultIngressClass, + }, + }, + Username: name, + } + } + + originalCache := cacheStoresFromObjs(t, someConsumer("valid")) + kongClient, err := NewKongClient( + zapr.NewLogger(zap.NewNop()), + time.Second, + util.ConfigDumpDiagnostic{}, + sendconfig.Config{ + FallbackConfiguration: true, + }, + mocks.NewEventRecorder(), + dpconf.DBModeOff, + clientsProvider, + updateStrategyResolver, + configChangeDetector, + lastValidConfigFetcher, + configBuilder, + originalCache, + fallbackConfigGenerator, + ) + require.NoError(t, err) + + t.Log("Calling KongClient.Update") + require.NoError(t, kongClient.Update(ctx)) + + t.Log("Verifying that the config builder cache was updated once") + require.Len(t, configBuilder.updateCacheCalls, 1) + + t.Log("Calling KongClient.Update again") + require.NoError(t, kongClient.Update(ctx)) + + t.Log("Verifying that the config builder cache was not updated when config was not changed") + require.Len(t, configBuilder.updateCacheCalls, 1) +} + func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { ctx := context.Background() gwClient := mustSampleGatewayClient(t) diff --git a/internal/store/cache_stores_snapshot.go b/internal/store/cache_stores_snapshot.go index b58263df36..f61afe25a9 100644 --- a/internal/store/cache_stores_snapshot.go +++ b/internal/store/cache_stores_snapshot.go @@ -14,6 +14,8 @@ import ( ) // TakeSnapshot takes a snapshot of the CacheStores. +// +// Deprecated: use TakeSnapshotIfChanged instead. func (c CacheStores) TakeSnapshot() (CacheStores, error) { // Create a fresh CacheStores instance to store the snapshot // in the c.takeSnapshot method. It happens here because it's @@ -97,7 +99,7 @@ func (c CacheStores) TakeSnapshotIfChanged(previousSnapshotHash SnapshotHash) ( return string(uid) + resourceVer }) if capturedErr != nil { - return CacheStores{}, "", capturedErr + return CacheStores{}, SnapshotHashEmpty, capturedErr } // Strings have to be used instead of byte slices, because Cmp.Ordered has to be satisfied. slices.Sort(valuesForHashComputation) @@ -110,12 +112,12 @@ func (c CacheStores) TakeSnapshotIfChanged(previousSnapshotHash SnapshotHash) ( // If the hash of the current state is the same as the hash of the previous snapshot, return an empty snapshot. if newHash == previousSnapshotHash { - return CacheStores{}, "", nil + return CacheStores{}, SnapshotHashEmpty, nil } // Take a snapshot of the current state as the hash of the current state differs from the previous one. if err := takeSnapshot(&snapshot, listOfStores); err != nil { - return CacheStores{}, "", fmt.Errorf("failed to take snapshot: %w", err) + return CacheStores{}, SnapshotHashEmpty, fmt.Errorf("failed to take snapshot: %w", err) } return snapshot, newHash, nil } From 5ac3bd26c3f80402196c330ab78cd6d1358798c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Wed, 29 May 2024 09:16:40 +0200 Subject: [PATCH 10/48] feat: backfill broken objects from last valid cache state (#6098) Adds the --use-last-valid-config-for-fallback flag that changes the default behavior of FallbackConfiguration feature by backfilling broken objects using the last valid config cache instead of only excluding those. Co-authored-by: Jakub Warczarek --- CHANGELOG.md | 4 +- FEATURE_GATES.md | 1 + docs/cli-arguments.md | 1 + internal/dataplane/fallback/fallback.go | 40 ++- internal/dataplane/fallback/fallback_test.go | 232 ++++++++++++++- internal/dataplane/fallback/graph.go | 11 + internal/dataplane/fallback/graph_test.go | 5 + internal/dataplane/fallback/helpers_test.go | 18 ++ internal/dataplane/kong_client.go | 57 +++- internal/dataplane/kong_client_test.go | 284 ++++++++++++++----- internal/dataplane/sendconfig/kong.go | 4 + internal/manager/config.go | 2 + internal/store/cache_stores_snapshot.go | 2 - 13 files changed, 568 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fb3961031..5750172e86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,7 +142,9 @@ Adding a new version? You'll need three changes: [#6010](https://github.com/Kong/kubernetes-ingress-controller/pull/6010) [#6047](https://github.com/Kong/kubernetes-ingress-controller/pull/6047) [#6071](https://github.com/Kong/kubernetes-ingress-controller/pull/6071) - +- Added `--use-last-valid-config-for-fallback` CLI flag to enable using the last valid configuration cache + to backfill excluded broken objects when the `FallbackConfiguration` feature gate is enabled. + [#6098](https://github.com/Kong/kubernetes-ingress-controller/pull/6098) - Add support for Kubernetes Gateway API v1.1: - add a flag `--enable-controller-gwapi-grpcroute` to control whether enable or disable GRPCRoute controller. - add support for `GRPCRoute` v1, which requires users to upgrade the Gateway API's CRD to v1.1. diff --git a/FEATURE_GATES.md b/FEATURE_GATES.md index 886d1ec173..ae7c071ea8 100644 --- a/FEATURE_GATES.md +++ b/FEATURE_GATES.md @@ -70,6 +70,7 @@ Features that reach GA and over time become stable will be removed from this tab | RewriteURIs | `false` | Alpha | 2.12.0 | TBD | | KongServiceFacade | `false` | Alpha | 3.1.0 | TBD | | SanitizeKonnectConfigDumps | `true` | Beta | 3.1.0 | TBD | +| FallbackConfiguration | `false` | Alpha | 3.2.0 | TBD | **NOTE**: The `Gateway` feature gate refers to [Gateway API](https://github.com/kubernetes-sigs/gateway-api) APIs which are in diff --git a/docs/cli-arguments.md b/docs/cli-arguments.md index 80e01a6ccb..73a29e0fcf 100644 --- a/docs/cli-arguments.md +++ b/docs/cli-arguments.md @@ -91,4 +91,5 @@ | `--term-delay` | `duration` | The time delay to sleep before SIGTERM or SIGINT will shut down the ingress controller. | `0s` | | `--update-status` | `bool` | Indicates if the ingress controller should update the status of resources (e.g. IP/Hostname for v1.Ingress, etc.). | `true` | | `--update-status-queue-buffer-size` | `int` | Buffer size of the underlying channels used to update the status of resources. | `8192` | +| `--use-last-valid-config-for-fallback` | `bool` | When recovering from config push failures, use the last valid configuration cache to backfill broken objects. | `false` | | `--watch-namespace` | `strings` | Namespace(s) in comma-separated format (or specify this flag multiple times) to watch for Kubernetes resources. Defaults to all namespaces. | `[]` | diff --git a/internal/dataplane/fallback/fallback.go b/internal/dataplane/fallback/fallback.go index add1eea399..463610a666 100644 --- a/internal/dataplane/fallback/fallback.go +++ b/internal/dataplane/fallback/fallback.go @@ -37,7 +37,7 @@ func (g *Generator) GenerateExcludingBrokenObjects( return store.CacheStores{}, fmt.Errorf("failed to build cache graph: %w", err) } - fallbackCache, _, err := cache.TakeSnapshotIfChanged(store.SnapshotHashEmpty) + fallbackCache, err := cache.TakeSnapshot() if err != nil { return store.CacheStores{}, fmt.Errorf("failed to take cache snapshot: %w", err) } @@ -61,3 +61,41 @@ func (g *Generator) GenerateExcludingBrokenObjects( return fallbackCache, nil } + +func (g *Generator) GenerateBackfillingBrokenObjects( + currentCache store.CacheStores, + lastValidCacheSnapshot store.CacheStores, + brokenObjects []ObjectHash, +) (store.CacheStores, error) { + // Generate a fallback cache snapshot excluding the broken objects. + fallbackCache, err := g.GenerateExcludingBrokenObjects(currentCache, brokenObjects) + if err != nil { + return store.CacheStores{}, fmt.Errorf("failed to generate fallback cache: %w", err) + } + + // Build a graph from the last valid cache snapshot. + lastValidGraph, err := g.cacheGraphProvider.CacheToGraph(lastValidCacheSnapshot) + if err != nil { + return store.CacheStores{}, fmt.Errorf("failed to build cache graph: %w", err) + } + + // Backfill the broken objects from the last valid cache snapshot. + for _, brokenObject := range brokenObjects { + objectsToBackfill, err := lastValidGraph.SubgraphObjects(brokenObject) + if err != nil { + return store.CacheStores{}, fmt.Errorf("failed to find dependants for %s: %w", brokenObject, err) + } + + for _, obj := range objectsToBackfill { + if err := fallbackCache.Add(obj); err != nil { + return store.CacheStores{}, fmt.Errorf("failed to add %s to the cache: %w", GetObjectHash(obj), err) + } + g.logger.V(util.DebugLevel).Info("Backfilled object to fallback cache", + "object_kind", obj.GetObjectKind(), + "object_name", obj.GetName(), + "object_namespace", obj.GetNamespace(), + ) + } + } + return fallbackCache, nil +} diff --git a/internal/dataplane/fallback/fallback_test.go b/internal/dataplane/fallback/fallback_test.go index cce0a9eb36..31adb8a97d 100644 --- a/internal/dataplane/fallback/fallback_test.go +++ b/internal/dataplane/fallback/fallback_test.go @@ -1,25 +1,58 @@ package fallback_test import ( + "errors" "testing" "github.com/go-logr/logr" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" + kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" + incubatorv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/incubator/v1alpha1" ) // mockGraphProvider is a mock implementation of the CacheGraphProvider interface. type mockGraphProvider struct { - graph *fallback.ConfigGraph - lastCalledWithStore store.CacheStores + graphToReturnOn map[store.CacheStores]*fallback.ConfigGraph + cacheToGraphCalls []store.CacheStores } // CacheToGraph returns the graph that was set on the mockGraphProvider. It also records the last store that was passed to it. func (m *mockGraphProvider) CacheToGraph(s store.CacheStores) (*fallback.ConfigGraph, error) { - m.lastCalledWithStore = s - return m.graph, nil + m.cacheToGraphCalls = append(m.cacheToGraphCalls, s) + if g, ok := m.graphToReturnOn[s]; ok { + return g, nil + } + return nil, errors.New("unexpected call") +} + +// ReturnGraphOn sets the graph that should be returned when CacheToGraph is called with the given cache stores. +func (m *mockGraphProvider) ReturnGraphOn(cache store.CacheStores, graph *fallback.ConfigGraph) { + if m.graphToReturnOn == nil { + m.graphToReturnOn = make(map[store.CacheStores]*fallback.ConfigGraph) + } + m.graphToReturnOn[cache] = graph +} + +// CacheToGraphLastCalledWith returns the last cache stores that were passed to CacheToGraph. +func (m *mockGraphProvider) CacheToGraphLastCalledWith() store.CacheStores { + if len(m.cacheToGraphCalls) == 0 { + return store.CacheStores{} + } + return m.cacheToGraphCalls[len(m.cacheToGraphCalls)-1] +} + +// CacheToGraphLastNCalledWith returns the last N cache stores that were passed to CacheToGraph. +func (m *mockGraphProvider) CacheToGraphLastNCalledWith(n int) []store.CacheStores { + if maxLen := len(m.cacheToGraphCalls); n > maxLen { + n = maxLen + } + return m.cacheToGraphCalls[len(m.cacheToGraphCalls)-n:] } func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { @@ -53,13 +86,14 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { Build() require.NoError(t, err) - graphProvider := &mockGraphProvider{graph: graph} + graphProvider := &mockGraphProvider{} + graphProvider.ReturnGraphOn(inputCacheStores, graph) g := fallback.NewGenerator(graphProvider, logr.Discard()) t.Run("ingressClass is broken", func(t *testing.T) { fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass)}) require.NoError(t, err) - require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") + require.Equal(t, inputCacheStores, graphProvider.CacheToGraphLastCalledWith(), "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) require.Empty(t, fallbackCache.IngressClassV1.List(), "ingressClass should be excluded as it's broken") require.Empty(t, fallbackCache.Service.List(), "service should be excluded as it depends on ingressClass") @@ -70,7 +104,7 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { t.Run("service is broken", func(t *testing.T) { fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(service)}) require.NoError(t, err) - require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") + require.Equal(t, inputCacheStores, graphProvider.CacheToGraphLastCalledWith(), "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) require.Empty(t, fallbackCache.Service.List(), "service should be excluded as it's broken") require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade should be excluded as it depends on service") @@ -81,7 +115,7 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { t.Run("serviceFacade is broken", func(t *testing.T) { fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(serviceFacade)}) require.NoError(t, err) - require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") + require.Equal(t, inputCacheStores, graphProvider.CacheToGraphLastCalledWith(), "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade should be excluded as it's broken") require.ElementsMatch(t, fallbackCache.IngressClassV1.List(), []any{ingressClass}, "ingressClass shouldn't be excluded as it doesn't depend on service") @@ -92,7 +126,7 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { t.Run("plugin is broken", func(t *testing.T) { fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(plugin)}) require.NoError(t, err) - require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") + require.Equal(t, inputCacheStores, graphProvider.CacheToGraphLastCalledWith(), "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) require.Empty(t, fallbackCache.Plugin.List(), "plugin should be excluded as it's broken") require.ElementsMatch(t, fallbackCache.IngressClassV1.List(), []any{ingressClass}, "ingressClass shouldn't be excluded as it doesn't depend on plugin") @@ -103,7 +137,7 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { t.Run("multiple objects are broken", func(t *testing.T) { fallbackCache, err := g.GenerateExcludingBrokenObjects(inputCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass), fallback.GetObjectHash(service)}) require.NoError(t, err) - require.Equal(t, inputCacheStores, graphProvider.lastCalledWithStore, "expected the generator to call CacheToGraph with the input cache stores") + require.Equal(t, inputCacheStores, graphProvider.CacheToGraphLastCalledWith(), "expected the generator to call CacheToGraph with the input cache stores") require.NotSame(t, inputCacheStores, fallbackCache) require.Empty(t, fallbackCache.IngressClassV1.List(), "ingressClass should be excluded as it's broken") require.Empty(t, fallbackCache.Service.List(), "service should be excluded as it's broken") @@ -111,3 +145,181 @@ func TestGenerator_GenerateExcludingBrokenObjects(t *testing.T) { require.ElementsMatch(t, fallbackCache.Plugin.List(), []any{plugin}, "plugin shouldn't be excluded as it doesn't depend on either ingressClass or service") }) } + +func TestGenerator_GenerateBackfillingBrokenObjects(t *testing.T) { + // We have to use real-world object types here as we're testing integration with store.CacheStores. + ingressClass := testIngressClass(t, "ingressClass") + service := testService(t, "service") + serviceFacade := testKongServiceFacade(t, "serviceFacade") + plugin := testKongPlugin(t, "kongPlugin") + inputCacheStores := cacheStoresFromObjs(t, ingressClass, service, serviceFacade, plugin) + + // We'll annotate the last valid objects, so we can verify that they were recovered from the last valid cache snapshot. + const lastValidAnnotationKey = "from-last-valid" + annotatedLastValid := func(o client.Object) client.Object { + o.SetAnnotations(map[string]string{lastValidAnnotationKey: "true"}) + return o + } + requireAnnotatedLastValid := func(t *testing.T, o client.Object) { + require.Equal(t, "true", o.GetAnnotations()[lastValidAnnotationKey], "expected the object to be recovered from the last valid cache snapshot") + } + requireNotAnnotatedLastValid := func(t *testing.T, o client.Object) { + require.NotContains(t, o.GetAnnotations(), lastValidAnnotationKey, "expected the object to not be recovered from the last valid cache snapshot") + } + + lastValidIngressClass := annotatedLastValid(testIngressClass(t, "ingressClass")) + lastValidService := annotatedLastValid(testService(t, "service")) + lastValidPlugin := annotatedLastValid(testKongPlugin(t, "kongPlugin")) + lastValidCacheStores := cacheStoresFromObjs(t, lastValidIngressClass, lastValidService, lastValidPlugin) + + // This graph doesn't reflect real dependencies between the objects - it's only used for testing purposes. + // It will be injected into the Generator via the mockGraphProvider. + // Dependency resolving between the objects is tested in TestResolveDependencies_* tests. + // + // Graph structure (edges define dependency -> dependant relationship): + // ┌────────────┐ ┌──────┐ + // │ingressClass│ │plugin│ + // └──────┬─────┘ └──────┘ + // │ + // ┌───▼───┐ + // │service│ + // └───┬───┘ + // │ + // ┌──────▼──────┐ + // │serviceFacade│ + // └─────────────┘ + graph, err := NewGraphBuilder(). + WithVertices(ingressClass, service, serviceFacade, plugin). + WithEdge(ingressClass, service). + WithEdge(service, serviceFacade). + Build() + require.NoError(t, err) + + // Fallback graph differs from the input graph by lack of the serviceFacade. + // ┌────────────┐ ┌──────┐ + // │ingressClass│ │plugin│ + // └──────┬─────┘ └──────┘ + // │ + // ┌───▼───┐ + // │service│ + // └───────┘ + lastValidGraph, err := NewGraphBuilder(). + WithVertices(lastValidIngressClass, lastValidService, lastValidPlugin). + WithEdge(lastValidIngressClass, lastValidService). + Build() + require.NoError(t, err) + + graphProvider := &mockGraphProvider{} + graphProvider.ReturnGraphOn(inputCacheStores, graph) + graphProvider.ReturnGraphOn(lastValidCacheStores, lastValidGraph) + g := fallback.NewGenerator(graphProvider, logr.Discard()) + + t.Run("ingressClass is broken", func(t *testing.T) { + fallbackCache, err := g.GenerateBackfillingBrokenObjects(inputCacheStores, lastValidCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass)}) + require.NoError(t, err) + require.Equal(t, []store.CacheStores{inputCacheStores, lastValidCacheStores}, graphProvider.CacheToGraphLastNCalledWith(2), + "expected the generator to call CacheToGraph with the input cache stores and the last valid cache stores") + require.NotSame(t, inputCacheStores, fallbackCache) + + fallbackIngressClass, err := getFromStore[*netv1.IngressClass](fallbackCache.IngressClassV1, ingressClass) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackIngressClass) + + fallbackService, err := getFromStore[*corev1.Service](fallbackCache.Service, service) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackService) + + require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade shouldn't be recovered as it wasn't in the last valid cache snapshot") + + fallbackPlugin, err := getFromStore[*kongv1.KongPlugin](fallbackCache.Plugin, plugin) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackPlugin) + }) + + t.Run("service is broken", func(t *testing.T) { + fallbackCache, err := g.GenerateBackfillingBrokenObjects(inputCacheStores, lastValidCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(service)}) + require.NoError(t, err) + require.Equal(t, []store.CacheStores{inputCacheStores, lastValidCacheStores}, graphProvider.CacheToGraphLastNCalledWith(2), + "expected the generator to call CacheToGraph with the input cache stores and fallback cache") + require.NotSame(t, inputCacheStores, fallbackCache) + + fallbackService, err := getFromStore[*corev1.Service](fallbackCache.Service, service) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackService) + + require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade shouldn't be recovered as it wasn't in the last valid cache snapshot") + + fallbackPlugin, err := getFromStore[*kongv1.KongPlugin](fallbackCache.Plugin, plugin) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackPlugin) + + fallbackIngressClass, err := getFromStore[*netv1.IngressClass](fallbackCache.IngressClassV1, ingressClass) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackIngressClass) + }) + + t.Run("serviceFacade is broken", func(t *testing.T) { + fallbackCache, err := g.GenerateBackfillingBrokenObjects(inputCacheStores, lastValidCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(serviceFacade)}) + require.NoError(t, err) + require.Equal(t, []store.CacheStores{inputCacheStores, lastValidCacheStores}, graphProvider.CacheToGraphLastNCalledWith(2), "expected the generator to call CacheToGraph with the input cache stores") + require.NotSame(t, inputCacheStores, fallbackCache) + + require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade should be excluded as it's broken and it's not present in the last valid cache snapshot") + + fallbackIngressClass, err := getFromStore[*netv1.IngressClass](fallbackCache.IngressClassV1, ingressClass) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackIngressClass) + + fallbackService, err := getFromStore[*corev1.Service](fallbackCache.Service, service) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackService) + + fallbackPlugin, err := getFromStore[*kongv1.KongPlugin](fallbackCache.Plugin, plugin) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackPlugin) + }) + + t.Run("plugin is broken", func(t *testing.T) { + fallbackCache, err := g.GenerateBackfillingBrokenObjects(inputCacheStores, lastValidCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(plugin)}) + require.NoError(t, err) + require.Equal(t, []store.CacheStores{inputCacheStores, lastValidCacheStores}, graphProvider.CacheToGraphLastNCalledWith(2), "expected the generator to call CacheToGraph with the input cache stores") + require.NotSame(t, inputCacheStores, fallbackCache) + + fallbackPlugin, err := getFromStore[*kongv1.KongPlugin](fallbackCache.Plugin, plugin) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackPlugin) + + fallbackIngressClass, err := getFromStore[*netv1.IngressClass](fallbackCache.IngressClassV1, ingressClass) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackIngressClass) + + fallbackService, err := getFromStore[*corev1.Service](fallbackCache.Service, service) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackService) + + fallbackServiceFacade, err := getFromStore[*incubatorv1alpha1.KongServiceFacade](fallbackCache.KongServiceFacade, serviceFacade) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackServiceFacade) + }) + + t.Run("multiple objects are broken", func(t *testing.T) { + fallbackCache, err := g.GenerateBackfillingBrokenObjects(inputCacheStores, lastValidCacheStores, []fallback.ObjectHash{fallback.GetObjectHash(ingressClass), fallback.GetObjectHash(service)}) + require.NoError(t, err) + require.Equal(t, []store.CacheStores{inputCacheStores, lastValidCacheStores}, graphProvider.CacheToGraphLastNCalledWith(2), "expected the generator to call CacheToGraph with the input cache stores") + require.NotSame(t, inputCacheStores, fallbackCache) + + fallbackIngressClass, err := getFromStore[*netv1.IngressClass](fallbackCache.IngressClassV1, ingressClass) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackIngressClass) + + fallbackService, err := getFromStore[*corev1.Service](fallbackCache.Service, service) + require.NoError(t, err) + requireAnnotatedLastValid(t, fallbackService) + + require.Empty(t, fallbackCache.KongServiceFacade.List(), "serviceFacade shouldn't be recovered as it wasn't in the last valid cache snapshot") + + fallbackPlugin, err := getFromStore[*kongv1.KongPlugin](fallbackCache.Plugin, plugin) + require.NoError(t, err) + requireNotAnnotatedLastValid(t, fallbackPlugin) + }) +} diff --git a/internal/dataplane/fallback/graph.go b/internal/dataplane/fallback/graph.go index a6dbe50eda..2986995b35 100644 --- a/internal/dataplane/fallback/graph.go +++ b/internal/dataplane/fallback/graph.go @@ -1,6 +1,7 @@ package fallback import ( + "errors" "fmt" "github.com/dominikbraun/graph" @@ -88,7 +89,17 @@ func (g *ConfigGraph) AdjacencyMap() (AdjacencyMap, error) { // SubgraphObjects returns all objects in the graph reachable from the source object, including the source object. // It uses a depth-first search to traverse the graph. +// If the source object is not in the graph, no objects are returned. func (g *ConfigGraph) SubgraphObjects(sourceHash ObjectHash) ([]client.Object, error) { + // First, ensure the source object is in the graph. + if _, err := g.graph.Vertex(sourceHash); err != nil { + // If the source object is not in the graph, return an empty list. + if errors.Is(err, graph.ErrVertexNotFound) { + return nil, nil + } + return nil, fmt.Errorf("failed to get source object from the graph: %w", err) + } + var objects []client.Object if err := graph.DFS(g.graph, sourceHash, func(hash ObjectHash) bool { obj, err := g.graph.Vertex(hash) diff --git a/internal/dataplane/fallback/graph_test.go b/internal/dataplane/fallback/graph_test.go index 74338766f3..aeeb0701ab 100644 --- a/internal/dataplane/fallback/graph_test.go +++ b/internal/dataplane/fallback/graph_test.go @@ -19,6 +19,7 @@ func TestConfigGraph_SubgraphObjects(t *testing.T) { F = NewMockObject("F") G = NewMockObject("G") H = NewMockObject("H") + I = NewMockObject("I") // Not included in the graph. ) // Graph structure (edges define dependency -> dependant relationship): @@ -80,4 +81,8 @@ func TestConfigGraph_SubgraphObjects(t *testing.T) { objects, err = g.SubgraphObjects(fallback.GetObjectHash(H)) require.NoError(t, err) require.ElementsMatch(t, []client.Object{H}, objects) + + objects, err = g.SubgraphObjects(fallback.GetObjectHash(I)) + require.NoError(t, err) + require.Empty(t, objects, "expected no objects returned for a source object not in the graph") } diff --git a/internal/dataplane/fallback/helpers_test.go b/internal/dataplane/fallback/helpers_test.go index 8d7e86be74..a46a370954 100644 --- a/internal/dataplane/fallback/helpers_test.go +++ b/internal/dataplane/fallback/helpers_test.go @@ -1,6 +1,7 @@ package fallback_test import ( + "errors" "fmt" "testing" @@ -8,6 +9,7 @@ import ( netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback" @@ -163,3 +165,19 @@ func NewMockObject(name string) *MockObject { func (m *MockObject) DeepCopyObject() runtime.Object { return m } + +// getFromStore retrieves an object of type T from the given cache store. +func getFromStore[T client.Object](c cache.Store, obj client.Object) (client.Object, error) { + o, exists, err := c.Get(obj) + if err != nil { + return nil, fmt.Errorf("failed to get object: %w", err) + } + if !exists { + return nil, errors.New("object not found") + } + typedObject, ok := o.(T) + if !ok { + return nil, fmt.Errorf("expected object of type %T, got %T", typedObject, o) + } + return typedObject, nil +} diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 527a9edd84..2d979186d1 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -63,6 +63,11 @@ type KongConfigBuilder interface { // FallbackConfigGenerator generates a fallback configuration based on a cache snapshot and a set of broken objects. type FallbackConfigGenerator interface { GenerateExcludingBrokenObjects(store.CacheStores, []fallback.ObjectHash) (store.CacheStores, error) + GenerateBackfillingBrokenObjects( + currentCache store.CacheStores, + lastValidCache store.CacheStores, + brokenObjects []fallback.ObjectHash, + ) (store.CacheStores, error) } // KongClient is a threadsafe high level API client for the Kong data-plane(s) @@ -155,6 +160,14 @@ type KongClient struct { // lastProcessedSnapshotHash stores the hash of the last processed Kubernetes objects cache snapshot. It's used to determine configuration // changes. Please note it is always empty when the `FallbackConfiguration` feature gate is turned off. lastProcessedSnapshotHash store.SnapshotHash + + // lastValidCacheSnapshot stores the state of the cache that was last successfully synced with the gateways. + // Please note it is only populated when the `FallbackConfiguration` feature gate is turned on and the + // `--use-last-valid-config-for-fallback` flag is set. + // lastValidCacheSnapshot and lastProcessedSnapshotHash do not always keep values related to the same cache snapshot. + // While lastProcessedSnapshotHash keeps track of the last processed cache snapshot (the one kept in KongClient.cache), + // lastValidCacheSnapshot can also represent the fallback cache snapshot that was successfully synced with gateways. + lastValidCacheSnapshot store.CacheStores } // NewKongClient provides a new KongClient object after connecting to the @@ -469,6 +482,9 @@ func (c *KongClient) Update(ctx context.Context) error { return gatewaysSyncErr } + // Gateways were successfully synced with the current configuration, so we can update the last valid cache snapshot. + c.maybePreserveTheLastValidConfigCache(cacheSnapshot) + // report on configured Kubernetes objects if enabled if c.AreKubernetesObjectReportsEnabled() { // if the configuration SHAs that have just been pushed are different than @@ -484,6 +500,15 @@ func (c *KongClient) Update(ctx context.Context) error { return nil } +// maybePreserveTheLastValidConfigCache preserves the last valid configuration cache if the `FallbackConfiguration` +// feature gate is enabled and the `--enable-last-valid-config-fallback` flag is set. +func (c *KongClient) maybePreserveTheLastValidConfigCache(lastValidCache store.CacheStores) { + if c.kongConfig.FallbackConfiguration && c.kongConfig.UseLastValidConfigForFallback { + c.logger.V(util.DebugLevel).Info("Preserving the last valid configuration cache") + c.lastValidCacheSnapshot = lastValidCache + } +} + // tryRecoveringFromGatewaysSyncError tries to recover from a configuration rejection by: // 1. Generating a fallback configuration and pushing it to the gateways if FallbackConfiguration feature is enabled. // 2. Applying the last valid configuration to the gateways if FallbackConfiguration is disabled or fallback @@ -520,7 +545,7 @@ func (c *KongClient) tryRecoveringFromGatewaysSyncError( // configuration excluding affected objects from the cache. func (c *KongClient) tryRecoveringWithFallbackConfiguration( ctx context.Context, - cacheSnapshot store.CacheStores, + currentCache store.CacheStores, gatewaysSyncErr error, ) error { // Extract the broken objects from the update error and generate a fallback configuration excluding them. @@ -528,10 +553,9 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( if err != nil { return fmt.Errorf("failed to extract broken objects from update error: %w", err) } - fallbackCache, err := c.fallbackConfigGenerator.GenerateExcludingBrokenObjects( - cacheSnapshot, - brokenObjects, - ) + + // Generate a fallback cache snapshot. + fallbackCache, err := c.generateFallbackCache(currentCache, brokenObjects) if err != nil { return fmt.Errorf("failed to generate fallback configuration: %w", err) } @@ -555,9 +579,32 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( // If Konnect sync fails, we should log the error and carry on as it's not a critical error. c.logger.Error(konnectSyncErr, "Failed to sync fallback configuration with Konnect") } + + // Configuration was successfully recovered with the fallback configuration. Store the last valid configuration. + c.maybePreserveTheLastValidConfigCache(fallbackCache) return nil } +// generateFallbackCache generates a fallback configuration based on the current cache and a set of broken objects. +// It will either exclude the broken objects from the cache or backfill them from the last valid cache snapshot +// depending on the UseLastValidConfigForFallback flag. +func (c *KongClient) generateFallbackCache( + currentCache store.CacheStores, + brokenObjects []fallback.ObjectHash, +) (store.CacheStores, error) { + if c.kongConfig.UseLastValidConfigForFallback { + return c.fallbackConfigGenerator.GenerateBackfillingBrokenObjects( + currentCache, + c.lastValidCacheSnapshot, + brokenObjects, + ) + } + return c.fallbackConfigGenerator.GenerateExcludingBrokenObjects( + currentCache, + brokenObjects, + ) +} + // extractBrokenObjectsFromUpdateError. func extractBrokenObjectsFromUpdateError(err error) ([]fallback.ObjectHash, error) { var brokenObjects []client.Object diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index a5a88912d1..a5f5e4e957 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -291,8 +291,10 @@ func (m mockConfigurationChangeDetector) HasConfigurationChanged( // mockKongLastValidConfigFetcher is a mock implementation of FallbackConfigGenerator interface. type mockFallbackConfigGenerator struct { - GenerateExcludingBrokenObjectsCalledWith lo.Tuple2[store.CacheStores, []fallback.ObjectHash] - GenerateExcludingBrokenObjectsResult store.CacheStores + GenerateResult store.CacheStores + + GenerateExcludingBrokenObjectsCalledWith lo.Tuple2[store.CacheStores, []fallback.ObjectHash] + GenerateBackfillingBrokenObjectsCalledWith lo.Tuple3[store.CacheStores, store.CacheStores, []fallback.ObjectHash] } func newMockFallbackConfigGenerator() *mockFallbackConfigGenerator { @@ -304,7 +306,16 @@ func (m *mockFallbackConfigGenerator) GenerateExcludingBrokenObjects( hashes []fallback.ObjectHash, ) (store.CacheStores, error) { m.GenerateExcludingBrokenObjectsCalledWith = lo.T2(stores, hashes) - return m.GenerateExcludingBrokenObjectsResult, nil + return m.GenerateResult, nil +} + +func (m *mockFallbackConfigGenerator) GenerateBackfillingBrokenObjects( + currentStores store.CacheStores, + lastValidStores store.CacheStores, + brokenObjects []fallback.ObjectHash, +) (store.CacheStores, error) { + m.GenerateBackfillingBrokenObjectsCalledWith = lo.T3(currentStores, lastValidStores, brokenObjects) + return m.GenerateResult, nil } func TestKongClientUpdate_AllExpectedClientsAreCalledAndErrorIsPropagated(t *testing.T) { @@ -963,11 +974,8 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { gatewayClients: []*adminapi.Client{gwClient}, konnectClient: konnectClient, } - updateStrategyResolver := newMockUpdateStrategyResolver(t) configChangeDetector := mockConfigurationChangeDetector{hasConfigurationChanged: true} - configBuilder := newMockKongConfigBuilder() lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} - fallbackConfigGenerator := newMockFallbackConfigGenerator() // We'll use KongConsumer as an example of a broken object, but it could be any supported type // for the purpose of this test as the fallback config generator is mocked anyway. @@ -986,86 +994,127 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { validConsumer := someConsumer("valid") brokenConsumer := someConsumer("broken") originalCache := cacheStoresFromObjs(t, validConsumer, brokenConsumer) - kongClient, err := NewKongClient( - zapr.NewLogger(zap.NewNop()), - time.Second, - util.ConfigDumpDiagnostic{}, - sendconfig.Config{ - FallbackConfiguration: true, - }, - mocks.NewEventRecorder(), - dpconf.DBModeOff, - clientsProvider, - updateStrategyResolver, - configChangeDetector, - lastValidConfigFetcher, - configBuilder, - originalCache, - fallbackConfigGenerator, - ) - require.NoError(t, err) + lastValidCache := cacheStoresFromObjs(t, validConsumer) - t.Log("Setting update strategy to return an error on the first call to trigger fallback configuration generation") - updateStrategyResolver.returnSpecificErrorOnUpdate(gwClient.BaseRootURL(), sendconfig.NewUpdateError( - []failures.ResourceFailure{ - lo.Must(failures.NewResourceFailure("violated constraint", brokenConsumer)), + testCases := []struct { + name string + enableLastValidConfigFallback bool + expectGenerateExcludingBrokenObjectsCalled bool + expectGenerateBackfillingBrokenObjectsCalledWith bool + }{ + { + name: "last valid config is disabled", + enableLastValidConfigFallback: false, + expectGenerateExcludingBrokenObjectsCalled: true, }, - errors.New("error on update"), - )) - - t.Log("Setting the config builder to return KongState with the valid consumer only") - configBuilder.kongState = &kongstate.KongState{ - Consumers: []kongstate.Consumer{ - { - Consumer: kong.Consumer{ - Username: lo.ToPtr(validConsumer.Username), - }, - }, + { + name: "last valid config is enabled", + enableLastValidConfigFallback: true, + expectGenerateBackfillingBrokenObjectsCalledWith: true, }, } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + configBuilder := newMockKongConfigBuilder() + fallbackConfigGenerator := newMockFallbackConfigGenerator() + updateStrategyResolver := newMockUpdateStrategyResolver(t) + kongClient, err := NewKongClient( + zapr.NewLogger(zap.NewNop()), + time.Second, + util.ConfigDumpDiagnostic{}, + sendconfig.Config{ + FallbackConfiguration: true, + UseLastValidConfigForFallback: tc.enableLastValidConfigFallback, + }, + mocks.NewEventRecorder(), + dpconf.DBModeOff, + clientsProvider, + updateStrategyResolver, + configChangeDetector, + lastValidConfigFetcher, + configBuilder, + originalCache, + fallbackConfigGenerator, + ) + require.NoError(t, err) - t.Log("Setting the fallback config generator to return a snapshot excluding the broken consumer") - fallbackCacheStoresToBeReturned := cacheStoresFromObjs(t, validConsumer) - fallbackConfigGenerator.GenerateExcludingBrokenObjectsResult = fallbackCacheStoresToBeReturned + t.Log("Injecting the last valid cache snapshot to be used for recovery") + kongClient.lastValidCacheSnapshot = lastValidCache - t.Log("Calling KongClient.Update") - err = kongClient.Update(ctx) - require.Error(t, err) + t.Log("Setting update strategy to return an error on the first call to trigger fallback configuration generation") + updateStrategyResolver.returnSpecificErrorOnUpdate(gwClient.BaseRootURL(), sendconfig.NewUpdateError( + []failures.ResourceFailure{ + lo.Must(failures.NewResourceFailure("violated constraint", brokenConsumer)), + }, + errors.New("error on update"), + )) + + t.Log("Setting the config builder to return KongState with the valid consumer only") + configBuilder.kongState = &kongstate.KongState{ + Consumers: []kongstate.Consumer{ + { + Consumer: kong.Consumer{ + Username: lo.ToPtr(validConsumer.Username), + }, + }, + }, + } - t.Log("Verifying that the config builder cache was updated twice") - require.Len(t, configBuilder.updateCacheCalls, 2, - "expected cache to be updated with a snapshot twice: first with the initial cache snapshot, then with the fallback one") + t.Log("Setting the fallback config generator to return a snapshot excluding the broken consumer") + fallbackCacheStoresToBeReturned := cacheStoresFromObjs(t, validConsumer) + fallbackConfigGenerator.GenerateResult = fallbackCacheStoresToBeReturned - t.Log("Verifying that the first cache update contains both consumers") - firstCacheUpdate := configBuilder.updateCacheCalls[0] - require.NotEqual(t, originalCache, firstCacheUpdate, "expected cache to be updated with a new snapshot") - _, hasConsumer, err := firstCacheUpdate.Consumer.Get(brokenConsumer) - require.NoError(t, err) - require.True(t, hasConsumer, "expected consumer to be in the first cache snapshot") + t.Log("Calling KongClient.Update") + err = kongClient.Update(ctx) + require.Error(t, err) - t.Log("Verifying that the fallback config generator was called with the first cache snapshot and the broken object hash") - expectedGenerateExcludingBrokenObjectsArgs := lo.T2(firstCacheUpdate, []fallback.ObjectHash{fallback.GetObjectHash(brokenConsumer)}) - require.Equal(t, expectedGenerateExcludingBrokenObjectsArgs, fallbackConfigGenerator.GenerateExcludingBrokenObjectsCalledWith, - "expected fallback config generator to be called with the first cache snapshot and the broken object hash") + t.Log("Verifying that the config builder cache was updated twice") + require.Len(t, configBuilder.updateCacheCalls, 2, + "expected cache to be updated with a snapshot twice: first with the initial cache snapshot, then with the fallback one") - t.Log("Verifying that the second config builder cache update contains the fallback snapshot") - secondCacheUpdate := configBuilder.updateCacheCalls[1] - require.Equal(t, fallbackCacheStoresToBeReturned, secondCacheUpdate, - "expected cache to be updated with the fallback snapshot on second call") + t.Log("Verifying that the first cache update contains both consumers") + firstCacheUpdate := configBuilder.updateCacheCalls[0] + require.NotEqual(t, originalCache, firstCacheUpdate, "expected cache to be updated with a new snapshot") + _, hasConsumer, err := firstCacheUpdate.Consumer.Get(brokenConsumer) + require.NoError(t, err) + require.True(t, hasConsumer, "expected consumer to be in the first cache snapshot") - t.Log("Verifying that the update strategy was called twice for gateway and Konnect") - updateStrategyResolver.assertUpdateCalledForURLs( - []string{ - gwClient.BaseRootURL(), konnectClient.BaseRootURL(), - gwClient.BaseRootURL(), konnectClient.BaseRootURL(), - }, - "expected update to be called twice: first with the initial config, then with the fallback one", - ) + if tc.expectGenerateExcludingBrokenObjectsCalled { + t.Log("Verifying that the fallback config generator was called with the first cache snapshot and the broken object hash") + expectedGenerateExcludingBrokenObjectsArgs := lo.T2(firstCacheUpdate, []fallback.ObjectHash{fallback.GetObjectHash(brokenConsumer)}) + require.Equal(t, expectedGenerateExcludingBrokenObjectsArgs, fallbackConfigGenerator.GenerateExcludingBrokenObjectsCalledWith, + "expected fallback config generator to be called with the first cache snapshot and the broken object hash") + + require.Empty(t, fallbackConfigGenerator.GenerateBackfillingBrokenObjectsCalledWith) + } + if tc.expectGenerateBackfillingBrokenObjectsCalledWith { + t.Log("Verifying that the fallback config generator was called with the first and last valid cache snapshots and the broken object hash") + expectedGenerateBackfillingBrokenObjectsArgs := lo.T3(firstCacheUpdate, lastValidCache, []fallback.ObjectHash{fallback.GetObjectHash(brokenConsumer)}) + require.Equal(t, expectedGenerateBackfillingBrokenObjectsArgs, fallbackConfigGenerator.GenerateBackfillingBrokenObjectsCalledWith, + "expected fallback config generator to be called with the current and last valid cache snapshots and the broken object hash") + } + + t.Log("Verifying that the second config builder cache update contains the fallback snapshot") + secondCacheUpdate := configBuilder.updateCacheCalls[1] + require.Equal(t, fallbackCacheStoresToBeReturned, secondCacheUpdate, + "expected cache to be updated with the fallback snapshot on second call") + + t.Log("Verifying that the update strategy was called twice for gateway and Konnect") + updateStrategyResolver.assertUpdateCalledForURLs( + []string{ + gwClient.BaseRootURL(), konnectClient.BaseRootURL(), + gwClient.BaseRootURL(), konnectClient.BaseRootURL(), + }, + "expected update to be called twice: first with the initial config, then with the fallback one", + ) - t.Log("Verifying that the last valid config is updated with the config excluding the broken consumer") - lastValidConfig, _ := lastValidConfigFetcher.LastValidConfig() - require.Len(t, lastValidConfig.Consumers, 1) - require.Equal(t, validConsumer.Username, *lastValidConfig.Consumers[0].Username) + t.Log("Verifying that the last valid config is updated with the config excluding the broken consumer") + lastValidConfig, ok := lastValidConfigFetcher.LastValidConfig() + require.True(t, ok) + require.Len(t, lastValidConfig.Consumers, 1) + require.Equal(t, validConsumer.Username, *lastValidConfig.Consumers[0].Username) + }) + } } func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing.T) { @@ -1208,6 +1257,93 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { require.False(t, hasLastValidConfig, "expected no last valid config to be stored as no successful recovery happened") } +func TestKongClient_LastValidCacheSnapshot(t *testing.T) { + var ( + ctx = context.Background() + testKonnectClient = mustSampleKonnectClient(t) + testGatewayClient = mustSampleGatewayClient(t) + + clientsProvider = mockGatewayClientsProvider{ + gatewayClients: []*adminapi.Client{testGatewayClient}, + konnectClient: testKonnectClient, + } + + updateStrategyResolver = newMockUpdateStrategyResolver(t) + configChangeDetector = mockConfigurationChangeDetector{hasConfigurationChanged: true} + configBuilder = newMockKongConfigBuilder() + lastValidConfigFetcher = &mockKongLastValidConfigFetcher{} + originalCache = cacheStoresFromObjs(t) + fallbackConfigGenerator = newMockFallbackConfigGenerator() + ) + + testCases := []struct { + name string + fallbackConfigurationFeatureEnabled bool + useLastValidConfigForFallbackEnabled bool + expectLastValidCacheSnapshotToBeSet bool + }{ + { + name: "FallbackConfiguration=true, UseLastValidConfigForFallback=false", + fallbackConfigurationFeatureEnabled: true, + useLastValidConfigForFallbackEnabled: false, + expectLastValidCacheSnapshotToBeSet: false, + }, + { + name: "FallbackConfiguration=true, UseLastValidConfigForFallback=true", + fallbackConfigurationFeatureEnabled: true, + useLastValidConfigForFallbackEnabled: true, + expectLastValidCacheSnapshotToBeSet: true, + }, + { + name: "FallbackConfiguration=false, UseLastValidConfigForFallback=false", + fallbackConfigurationFeatureEnabled: false, + useLastValidConfigForFallbackEnabled: false, + expectLastValidCacheSnapshotToBeSet: false, + }, + { + name: "FallbackConfiguration=false, UseLastValidConfigForFallback=true", + fallbackConfigurationFeatureEnabled: false, + useLastValidConfigForFallbackEnabled: true, + expectLastValidCacheSnapshotToBeSet: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kongClient, err := NewKongClient( + zapr.NewLogger(zap.NewNop()), + time.Second, + util.ConfigDumpDiagnostic{}, + sendconfig.Config{ + FallbackConfiguration: tc.fallbackConfigurationFeatureEnabled, + UseLastValidConfigForFallback: tc.useLastValidConfigForFallbackEnabled, + }, + mocks.NewEventRecorder(), + dpconf.DBModeOff, + clientsProvider, + updateStrategyResolver, + configChangeDetector, + lastValidConfigFetcher, + configBuilder, + originalCache, + fallbackConfigGenerator, + ) + require.NoError(t, err) + + require.Empty(t, kongClient.lastValidCacheSnapshot, "expected last valid cache snapshot to be empty") + + err = kongClient.Update(ctx) + require.NoError(t, err) + + lastValid := kongClient.lastValidCacheSnapshot + if tc.expectLastValidCacheSnapshotToBeSet { + require.NotEmpty(t, lastValid, "expected last valid cache snapshot to be set after successful update") + } else { + require.Empty(t, lastValid, "expected last valid cache snapshot to remain empty") + } + }) + } +} + func cacheStoresFromObjs(t *testing.T, objs ...runtime.Object) store.CacheStores { for i := range objs { obj := objs[i].(client.Object) diff --git a/internal/dataplane/sendconfig/kong.go b/internal/dataplane/sendconfig/kong.go index 59bfb08336..1607d3b979 100644 --- a/internal/dataplane/sendconfig/kong.go +++ b/internal/dataplane/sendconfig/kong.go @@ -37,4 +37,8 @@ type Config struct { // FallbackConfiguration indicates whether to generate fallback configuration in the case of entity // errors returned by the Kong Admin API. FallbackConfiguration bool + + // UseLastValidConfigForFallback indicates whether to use the last valid config cache to backfill broken objects + // when recovering from a config push failure. + UseLastValidConfigForFallback bool } diff --git a/internal/manager/config.go b/internal/manager/config.go index bb4f53354b..aa1df49a73 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -56,6 +56,7 @@ type Config struct { KongWorkspace string AnonymousReports bool EnableReverseSync bool + UseLastValidConfigForFallback bool SyncPeriod time.Duration SkipCACertificates bool CacheSyncTimeout time.Duration @@ -179,6 +180,7 @@ func (c *Config) FlagSet() *pflag.FlagSet { flagSet.StringVar(&c.KongWorkspace, "kong-workspace", "", "Kong Enterprise workspace to configure. Leave this empty if not using Kong workspaces.") flagSet.BoolVar(&c.AnonymousReports, "anonymous-reports", true, `Send anonymized usage data to help improve Kong.`) flagSet.BoolVar(&c.EnableReverseSync, "enable-reverse-sync", false, `Send configuration to Kong even if the configuration checksum has not changed since previous update.`) + flagSet.BoolVar(&c.UseLastValidConfigForFallback, "use-last-valid-config-for-fallback", false, `When recovering from config push failures, use the last valid configuration cache to backfill broken objects.`) // Default has to be explicitly passed to generate the proper docs. See https://github.com/kubernetes-sigs/controller-runtime/blob/f1c5dd3851ce3df8b4b7830d9b6eae6271f6932d/pkg/cache/cache.go#L146-L151. flagSet.DurationVar(&c.SyncPeriod, "sync-period", 10*time.Hour, `Determine the minimum frequency at which watched resources are reconciled. Set to 0 to use default from controller-runtime.`) flagSet.BoolVar(&c.SkipCACertificates, "skip-ca-certificates", false, `Disable syncing CA certificate syncing (for use with multi-workspace environments).`) diff --git a/internal/store/cache_stores_snapshot.go b/internal/store/cache_stores_snapshot.go index f61afe25a9..08deccab66 100644 --- a/internal/store/cache_stores_snapshot.go +++ b/internal/store/cache_stores_snapshot.go @@ -14,8 +14,6 @@ import ( ) // TakeSnapshot takes a snapshot of the CacheStores. -// -// Deprecated: use TakeSnapshotIfChanged instead. func (c CacheStores) TakeSnapshot() (CacheStores, error) { // Create a fresh CacheStores instance to store the snapshot // in the c.takeSnapshot method. It happens here because it's From fc963f7e566a801478ff0fea7c3e40c0ec96a0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Wed, 29 May 2024 12:11:24 +0200 Subject: [PATCH 11/48] feat: emit fallback configuration events (#6099) --- CHANGELOG.md | 3 + internal/dataplane/kong_client.go | 63 ++++++-- internal/dataplane/kong_client_test.go | 195 +++++++++++++++++++++++++ test/mocks/events_recorder.go | 23 ++- 4 files changed, 262 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5750172e86..3bf4f8e1cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,9 @@ Adding a new version? You'll need three changes: - Added `--use-last-valid-config-for-fallback` CLI flag to enable using the last valid configuration cache to backfill excluded broken objects when the `FallbackConfiguration` feature gate is enabled. [#6098](https://github.com/Kong/kubernetes-ingress-controller/pull/6098) +- Added `FallbackKongConfigurationSucceeded`, `FallbackKongConfigurationTranslationFailed` and + `FallbackKongConfigurationApplyFailed` Kubernetes Events to report the status of the fallback configuration. + [#6099](https://github.com/Kong/kubernetes-ingress-controller/pull/6099) - Add support for Kubernetes Gateway API v1.1: - add a flag `--enable-controller-gwapi-grpcroute` to control whether enable or disable GRPCRoute controller. - add support for `GRPCRoute` v1, which requires users to upgrade the Gateway API's CRD to v1.1. diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 2d979186d1..187a1cbedd 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -48,6 +48,13 @@ const ( KongConfigurationTranslationFailedEventReason = "KongConfigurationTranslationFailed" // KongConfigurationApplyFailedEventReason defines an event reason used for creating all config apply resource failure events. KongConfigurationApplyFailedEventReason = "KongConfigurationApplyFailed" + + // FallbackKongConfigurationApplySucceededEventReason defines an event reason to tell the updating of fallback Kong configuration succeeded. + FallbackKongConfigurationApplySucceededEventReason = "FallbackKongConfigurationSucceeded" + // FallbackKongConfigurationTranslationFailedEventReason defines an event reason used for creating fallback translation resource failure events. + FallbackKongConfigurationTranslationFailedEventReason = "FallbackKongConfigurationTranslationFailed" + // FallbackKongConfigurationApplyFailedEventReason defines an event reason used for creating fallback config apply resource failure events. + FallbackKongConfigurationApplyFailedEventReason = "FallbackKongConfigurationApplyFailed" ) // ----------------------------------------------------------------------------- @@ -459,8 +466,9 @@ func (c *KongClient) Update(ctx context.Context) error { c.logger.V(util.DebugLevel).Info("Successfully built data-plane configuration") } - shas, gatewaysSyncErr := c.sendOutToGatewayClients(ctx, parsingResult.KongState, c.kongConfig) - konnectSyncErr := c.maybeSendOutToKonnectClient(ctx, parsingResult.KongState, c.kongConfig) + const isFallback = false + shas, gatewaysSyncErr := c.sendOutToGatewayClients(ctx, parsingResult.KongState, c.kongConfig, isFallback) + konnectSyncErr := c.maybeSendOutToKonnectClient(ctx, parsingResult.KongState, c.kongConfig, isFallback) // Taking into account the results of syncing configuration with Gateways and Konnect, and potential translation // failures, calculate the config status and update it. @@ -533,7 +541,8 @@ func (c *KongClient) tryRecoveringFromGatewaysSyncError( // If FallbackConfiguration is disabled, or we failed to recover using the fallback configuration, we should // apply the last valid configuration to the gateways. if state, found := c.kongConfigFetcher.LastValidConfig(); found { - if _, fallbackSyncErr := c.sendOutToGatewayClients(ctx, state, c.kongConfig); fallbackSyncErr != nil { + const isFallback = true + if _, fallbackSyncErr := c.sendOutToGatewayClients(ctx, state, c.kongConfig, isFallback); fallbackSyncErr != nil { return errors.Join(gatewaysSyncErr, fallbackSyncErr) } c.logger.V(util.DebugLevel).Info("Due to errors in the current config, the last valid config has been pushed to Gateways") @@ -564,17 +573,20 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( c.kongConfigBuilder.UpdateCache(fallbackCache) fallbackParsingResult := c.kongConfigBuilder.BuildKongConfig() - // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6081 - // Emit Kubernetes events depending on fallback configuration parsing result. + if failuresCount := len(fallbackParsingResult.TranslationFailures); failuresCount > 0 { + c.recordResourceFailureEvents(fallbackParsingResult.TranslationFailures, FallbackKongConfigurationTranslationFailedEventReason) + c.logger.V(util.DebugLevel).Info("Translation failures occurred when building fallback data-plane configuration", "count", failuresCount) + } // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6082 // Expose Prometheus metrics for fallback configuration parsing result. - _, gatewaysSyncErr = c.sendOutToGatewayClients(ctx, fallbackParsingResult.KongState, c.kongConfig) + const isFallback = true + _, gatewaysSyncErr = c.sendOutToGatewayClients(ctx, fallbackParsingResult.KongState, c.kongConfig, isFallback) if gatewaysSyncErr != nil { return fmt.Errorf("failed to sync fallback configuration with gateways: %w", gatewaysSyncErr) } - konnectSyncErr := c.maybeSendOutToKonnectClient(ctx, fallbackParsingResult.KongState, c.kongConfig) + konnectSyncErr := c.maybeSendOutToKonnectClient(ctx, fallbackParsingResult.KongState, c.kongConfig, isFallback) if konnectSyncErr != nil { // If Konnect sync fails, we should log the error and carry on as it's not a critical error. c.logger.Error(konnectSyncErr, "Failed to sync fallback configuration with Konnect") @@ -628,7 +640,10 @@ func extractBrokenObjectsFromUpdateError(err error) ([]fallback.ObjectHash, erro // sendOutToGatewayClients will generate deck content (config) from the provided kong state // and send it out to each of the configured gateway clients. func (c *KongClient) sendOutToGatewayClients( - ctx context.Context, s *kongstate.KongState, config sendconfig.Config, + ctx context.Context, + s *kongstate.KongState, + config sendconfig.Config, + isFallback bool, ) ([]string, error) { gatewayClients := c.clientsProvider.GatewayClients() if len(gatewayClients) == 0 { @@ -645,7 +660,7 @@ func (c *KongClient) sendOutToGatewayClients( c.logger.V(util.DebugLevel).Info("Sending configuration to gateway clients", "urls", configureGatewayClientURLs) shas, err := iter.MapErr(gatewayClientsToConfigure, func(client **adminapi.Client) (string, error) { - return c.sendToClient(ctx, *client, s, config) + return c.sendToClient(ctx, *client, s, config, isFallback) }) if err != nil { return nil, err @@ -673,14 +688,19 @@ func (c *KongClient) sendOutToGatewayClients( // maybeSendOutToKonnectClient sends out the configuration to Konnect when KonnectClient is provided. // It's a noop when Konnect integration is not enabled. -func (c *KongClient) maybeSendOutToKonnectClient(ctx context.Context, s *kongstate.KongState, config sendconfig.Config) error { +func (c *KongClient) maybeSendOutToKonnectClient( + ctx context.Context, + s *kongstate.KongState, + config sendconfig.Config, + isFallback bool, +) error { konnectClient := c.clientsProvider.KonnectClient() // There's no KonnectClient configured, that's totally fine. if konnectClient == nil { return nil } - if _, err := c.sendToClient(ctx, konnectClient, s, config); err != nil { + if _, err := c.sendToClient(ctx, konnectClient, s, config, isFallback); err != nil { // In case of an error, we only log it since we don't want the Konnect to affect the basic functionality // of the controller. @@ -723,6 +743,7 @@ func (c *KongClient) sendToClient( client sendconfig.AdminAPIClient, s *kongstate.KongState, config sendconfig.Config, + isFallback bool, ) (string, error) { logger := c.logger.WithValues("url", client.AdminAPIClient().BaseRootURL()) @@ -756,7 +777,7 @@ func (c *KongClient) sendToClient( // Only record events on applying configuration to Kong gateway here. // Nil error is expected to be passed to indicate success. if !client.IsKonnect() { - c.recordApplyConfigurationEvents(err, client.BaseRootURL()) + c.recordApplyConfigurationEvents(err, client.BaseRootURL(), isFallback) } if err != nil { var ( @@ -765,7 +786,11 @@ func (c *KongClient) sendToClient( responseParsingErr sendconfig.ResponseParsingError ) if errors.As(err, &updateErr) { - c.recordResourceFailureEvents(updateErr.ResourceFailures(), KongConfigurationApplyFailedEventReason) + reason := KongConfigurationApplyFailedEventReason + if isFallback { + reason = FallbackKongConfigurationApplyFailedEventReason + } + c.recordResourceFailureEvents(updateErr.ResourceFailures(), reason) } if errors.As(err, &responseParsingErr) { rawResponseBody = responseParsingErr.ResponseBody() @@ -915,7 +940,7 @@ func (c *KongClient) recordResourceFailureEvents(resourceFailures []failures.Res } // recordApplyConfigurationEvents records event attached to KIC pod after KIC applied Kong configuration. -func (c *KongClient) recordApplyConfigurationEvents(err error, rootURL string) { +func (c *KongClient) recordApplyConfigurationEvents(err error, rootURL string, isFallback bool) { podNN, ok := c.controllerPodReference.Get() if !ok { // Can't record an event without a controller pod reference to attach to. @@ -926,10 +951,20 @@ func (c *KongClient) recordApplyConfigurationEvents(err error, rootURL string) { reason := KongConfigurationApplySucceededEventReason message := fmt.Sprintf("successfully applied Kong configuration to %s", rootURL) + if isFallback { + reason = FallbackKongConfigurationApplySucceededEventReason + message = fmt.Sprintf("successfully applied fallback Kong configuration to %s", rootURL) + } + if err != nil { eventType = corev1.EventTypeWarning reason = KongConfigurationApplyFailedEventReason message = fmt.Sprintf("failed to apply Kong configuration to %s: %v", rootURL, err) + + if isFallback { + reason = FallbackKongConfigurationApplyFailedEventReason + message = fmt.Sprintf("failed to apply fallback Kong configuration to %s: %v", rootURL, err) + } } pod := &corev1.Pod{ diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index a5f5e4e957..931b0eb008 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -466,6 +466,12 @@ type mockKongConfigBuilder struct { translationFailuresToReturn []failures.ResourceFailure kongState *kongstate.KongState updateCacheCalls []store.CacheStores + + // onlyFirstCallWithNoTranslationFailures is used to simulate a scenario where the first call to the + // KongConfigBuilder has no translation failures, but subsequent calls do (e.g. to trigger translation failures in + // fallback configuration). + onlyFirstBuildCallWithNoTranslationFailures bool + buildCalled bool } func newMockKongConfigBuilder() *mockKongConfigBuilder { @@ -475,6 +481,13 @@ func newMockKongConfigBuilder() *mockKongConfigBuilder { } func (p *mockKongConfigBuilder) BuildKongConfig() translator.KongConfigBuildingResult { + if p.onlyFirstBuildCallWithNoTranslationFailures && !p.buildCalled { + p.buildCalled = true + return translator.KongConfigBuildingResult{ + KongState: p.kongState, + TranslationFailures: nil, + } + } return translator.KongConfigBuildingResult{ KongState: p.kongState, TranslationFailures: p.translationFailuresToReturn, @@ -506,6 +519,11 @@ func (p *mockKongConfigBuilder) returnTranslationFailures(enabled bool) { } } +func (p *mockKongConfigBuilder) returnTranslationFailuresForAllButFirstCall(failures []failures.ResourceFailure) { + p.onlyFirstBuildCallWithNoTranslationFailures = true + p.translationFailuresToReturn = failures +} + func TestKongClientUpdate_ConfigStatusIsNotified(t *testing.T) { var ( ctx = context.Background() @@ -653,6 +671,183 @@ func TestKongClient_ApplyConfigurationEvents(t *testing.T) { } } +func TestKongClient_KubernetesEvents(t *testing.T) { + t.Setenv("POD_NAMESPACE", "test-namespace") + t.Setenv("POD_NAME", "test-pod") + + ctx := context.Background() + testGatewayClient := mustSampleGatewayClient(t) + clientsProvider := mockGatewayClientsProvider{ + gatewayClients: []*adminapi.Client{testGatewayClient}, + } + configChangeDetector := mockConfigurationChangeDetector{hasConfigurationChanged: true} + testIngress := helpers.WithTypeMeta(t, &netv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "obj-1", + Namespace: "namespace", + }, + }) + testService := helpers.WithTypeMeta(t, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "obj-2", + Namespace: "namespace", + }, + }) + + testCases := []struct { + name string + fallbackConfiguration bool + translationFailures bool + updateError bool + entityErrors bool + fallbackConfigurationUpdateError bool + fallbackConfigurationTranslationFailures bool + expectError bool + expectEmittingEvents []string + }{ + { + name: "successful update", + expectError: false, + expectEmittingEvents: []string{ + "Normal KongConfigurationSucceeded", + }, + }, + { + name: "translation failures", + translationFailures: true, + expectError: false, + expectEmittingEvents: []string{ + "Ingress: Warning KongConfigurationTranslationFailed", + "Service: Warning KongConfigurationTranslationFailed", + "Pod: Normal KongConfigurationSucceeded", + }, + }, + { + name: "update error", + updateError: true, + expectError: true, + expectEmittingEvents: []string{ + "Pod: Warning KongConfigurationApplyFailed", + }, + }, + { + name: "update error with entity errors", + updateError: true, + entityErrors: true, + expectError: true, + expectEmittingEvents: []string{ + "Pod: Warning KongConfigurationApplyFailed", + "Ingress: Warning KongConfigurationApplyFailed", + "Service: Warning KongConfigurationApplyFailed", + }, + }, + { + name: "update error with entity errors, fallback configuration applied", + fallbackConfiguration: true, + updateError: true, + entityErrors: true, + expectError: true, + expectEmittingEvents: []string{ + "Pod: Warning KongConfigurationApplyFailed", + "Ingress: Warning KongConfigurationApplyFailed", + "Service: Warning KongConfigurationApplyFailed", + "Pod: Normal FallbackKongConfigurationSucceeded", + }, + }, + { + name: "update error with entity errors, fallback configuration failures", + fallbackConfiguration: true, + fallbackConfigurationUpdateError: true, + updateError: true, + entityErrors: true, + expectError: true, + expectEmittingEvents: []string{ + "Pod: Warning KongConfigurationApplyFailed", + "Ingress: Warning KongConfigurationApplyFailed", + "Service: Warning KongConfigurationApplyFailed", + "Pod: Warning FallbackKongConfigurationApplyFailed", + "Ingress: Warning FallbackKongConfigurationApplyFailed", + }, + }, + { + name: "update error with entity errors, fallback translation failures", + fallbackConfiguration: true, + updateError: true, + entityErrors: true, + expectError: true, + fallbackConfigurationTranslationFailures: true, + expectEmittingEvents: []string{ + "Pod: Warning KongConfigurationApplyFailed", + "Ingress: Warning KongConfigurationApplyFailed", + "Service: Warning KongConfigurationApplyFailed", + "Ingress: Warning FallbackKongConfigurationTranslationFailed", + "Service: Warning FallbackKongConfigurationTranslationFailed", + "Pod: Normal FallbackKongConfigurationSucceeded", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + updateStrategyResolver := newMockUpdateStrategyResolver(t) + configBuilder := newMockKongConfigBuilder() + eventRecorder := mocks.NewEventRecorder() + lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} + kongClient := setupTestKongClient(t, updateStrategyResolver, clientsProvider, configChangeDetector, configBuilder, eventRecorder, lastValidConfigFetcher) + kongClient.kongConfig.FallbackConfiguration = tc.fallbackConfiguration + + if tc.translationFailures { + configBuilder.translationFailuresToReturn = []failures.ResourceFailure{ + lo.Must(failures.NewResourceFailure("some reason", testIngress)), + lo.Must(failures.NewResourceFailure("some reason", testService)), + } + } + if tc.updateError { + if tc.entityErrors { + updateStrategyResolver.returnSpecificErrorOnUpdate(testGatewayClient.BaseRootURL(), sendconfig.NewUpdateError( + []failures.ResourceFailure{ + lo.Must(failures.NewResourceFailure("violated constraint", testIngress)), + lo.Must(failures.NewResourceFailure("violated constraint", testService)), + }, + errors.New("error on update"), + )) + } else { + updateStrategyResolver.returnErrorOnUpdate(testGatewayClient.BaseRootURL()) + } + } + if tc.updateError && tc.fallbackConfigurationUpdateError { + updateStrategyResolver.returnSpecificErrorOnUpdate(testGatewayClient.BaseRootURL(), sendconfig.NewUpdateError( + []failures.ResourceFailure{ + lo.Must(failures.NewResourceFailure("violated constraint", testIngress)), + }, errors.New("error on update"), + )) + } + if tc.fallbackConfigurationTranslationFailures { + configBuilder.returnTranslationFailuresForAllButFirstCall([]failures.ResourceFailure{ + lo.Must(failures.NewResourceFailure("some reason", testIngress)), + lo.Must(failures.NewResourceFailure("some reason", testService)), + }) + } + + err := kongClient.Update(ctx) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + emittedEvents := eventRecorder.Events() + require.Len(t, emittedEvents, len(tc.expectEmittingEvents)) + for _, expectedEvent := range tc.expectEmittingEvents { + containsExpectedEvent := lo.ContainsBy(emittedEvents, func(event string) bool { + return strings.Contains(event, expectedEvent) + }) + require.True(t, containsExpectedEvent, "expected event %q not found in %v", expectedEvent, eventRecorder.Events()) + } + }) + } +} + func TestKongClient_EmptyConfigUpdate(t *testing.T) { var ( ctx = context.Background() diff --git a/test/mocks/events_recorder.go b/test/mocks/events_recorder.go index 859716f6b1..9819ad3b26 100644 --- a/test/mocks/events_recorder.go +++ b/test/mocks/events_recorder.go @@ -5,6 +5,9 @@ import ( "sync" "k8s.io/apimachinery/pkg/runtime" + + "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/scheme" + "github.com/kong/kubernetes-ingress-controller/v3/internal/util" ) // EventRecorder is a mock implementation of the k8s.io/client-go/tools/record.EventRecorder interface. @@ -17,16 +20,16 @@ func NewEventRecorder() *EventRecorder { return &EventRecorder{} } -func (r *EventRecorder) Event(_ runtime.Object, eventtype, reason, message string) { - r.writeEvent(eventtype, reason, "%s", message) +func (r *EventRecorder) Event(o runtime.Object, eventtype, reason, message string) { + r.writeEvent(o, eventtype, reason, "%s", message) } -func (r *EventRecorder) Eventf(_ runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { - r.writeEvent(eventtype, reason, messageFmt, args...) +func (r *EventRecorder) Eventf(o runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { + r.writeEvent(o, eventtype, reason, messageFmt, args...) } -func (r *EventRecorder) AnnotatedEventf(_ runtime.Object, _ map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { - r.writeEvent(eventtype, reason, messageFmt, args...) +func (r *EventRecorder) AnnotatedEventf(o runtime.Object, _ map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { + r.writeEvent(o, eventtype, reason, messageFmt, args...) } func (r *EventRecorder) Events() []string { @@ -37,8 +40,12 @@ func (r *EventRecorder) Events() []string { return copied } -func (r *EventRecorder) writeEvent(eventtype, reason, messageFmt string, args ...interface{}) { +func (r *EventRecorder) writeEvent(o runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { r.l.Lock() defer r.l.Unlock() - r.events = append(r.events, fmt.Sprintf(eventtype+" "+reason+" "+messageFmt, args...)) + + s, _ := scheme.Get() + _ = util.PopulateTypeMeta(o, s) + fmtString := fmt.Sprintf("%s: %s %s %s", o.GetObjectKind().GroupVersionKind().Kind, eventtype, reason, messageFmt) + r.events = append(r.events, fmt.Sprintf(fmtString, args...)) } From 12909026204ff74e8b3b702dfa7281d6205853a1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 14:27:48 +0200 Subject: [PATCH 12/48] chore(deps): update dependency gke to v1.30.1 (#6083) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/test_dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/test_dependencies.yaml b/.github/test_dependencies.yaml index a38d8314c6..dc440dd7da 100644 --- a/.github/test_dependencies.yaml +++ b/.github/test_dependencies.yaml @@ -12,7 +12,7 @@ e2e: - 'v1.25.16' gke: # renovate: datasource=custom.gke-rapid depName=gke versioning=semver - - '1.30.0' + - '1.30.1' # For Istio, we define combinations of Kind and Istio versions that will be # used directly in the test matrix `include` section. From a2670f23f366b8e90193c785517c9399d577fda0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 14:41:27 +0200 Subject: [PATCH 13/48] chore(deps): bump github.com/kong/go-database-reconciler from 1.11.0 to 1.12.0 (#6093) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump github.com/kong/go-database-reconciler Bumps [github.com/kong/go-database-reconciler](https://github.com/kong/go-database-reconciler) from 1.11.0 to 1.12.0. - [Release notes](https://github.com/kong/go-database-reconciler/releases) - [Commits](https://github.com/kong/go-database-reconciler/compare/v1.11.0...v1.12.0) --- updated-dependencies: - dependency-name: github.com/kong/go-database-reconciler dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * tests: fix invalid expectation --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patryk Małek --- go.mod | 2 +- go.sum | 4 ++-- test/kongintegration/inmemory_update_strategy_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index efe1693145..a80bf3eed6 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/jpillora/backoff v1.0.0 - github.com/kong/go-database-reconciler v1.11.0 + github.com/kong/go-database-reconciler v1.12.0 github.com/kong/go-kong v0.55.0 github.com/kong/kubernetes-telemetry v0.1.3 github.com/kong/kubernetes-testing-framework v0.47.0 diff --git a/go.sum b/go.sum index 9a2e72550e..44da2e84c6 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kong/go-database-reconciler v1.11.0 h1:AowvG85lOWMZSCMZ9UpCGLAYoq/lXvoXJGc6BXH2+qU= -github.com/kong/go-database-reconciler v1.11.0/go.mod h1:fX9SV2ukbuUdVw2h1rakWTi/DUxAV9cbj4QWttoiRrc= +github.com/kong/go-database-reconciler v1.12.0 h1:8+mt2VX/j5uTByPF0dC7Xsh9LscaPjxxXErzaKbL2ik= +github.com/kong/go-database-reconciler v1.12.0/go.mod h1:fX9SV2ukbuUdVw2h1rakWTi/DUxAV9cbj4QWttoiRrc= github.com/kong/go-kong v0.55.0 h1:lonKRzsDGk12dh9E+y+pWnY2ThXhKuMHjzBHSpCvQLw= github.com/kong/go-kong v0.55.0/go.mod h1:i1cMgTu6RYPHSyMpviShddRnc+DML/vlpgKC00hr8kU= github.com/kong/kubernetes-telemetry v0.1.3 h1:Hz2tkHGIIUqbn1x46QRDmmNjbEtJyxyOvHSPne3uPto= diff --git a/test/kongintegration/inmemory_update_strategy_test.go b/test/kongintegration/inmemory_update_strategy_test.go index e58f5c9a46..8d1124bd20 100644 --- a/test/kongintegration/inmemory_update_strategy_test.go +++ b/test/kongintegration/inmemory_update_strategy_test.go @@ -90,7 +90,7 @@ func TestUpdateStrategyInMemory_PropagatesResourcesErrors(t *testing.T) { }, }, } - expectedMessage := "invalid service:test-service: failed conditional validation given value of field 'protocol'" + expectedMessage := "invalid path: value must be null" expectedRawErrBody := []byte(`{"code":14,"name":"invalid declarative configuration","fields":{},"message":"declarative config is invalid: {}","flattened_errors":[{"entity_type":"service","entity_name":"test-service","entity_tags":["k8s-name:test-service","k8s-namespace:default","k8s-kind:Service","k8s-uid:a3b8afcc-9f19-42e4-aa8f-5866168c2ad3","k8s-group:","k8s-version:v1"],"errors":[{"type":"field","message":"value must be null","field":"path"},{"type":"entity","message":"failed conditional validation given value of field 'protocol'"}],"entity":{"path":"/test","name":"test-service","protocol":"grpc","tags":["k8s-name:test-service","k8s-namespace:default","k8s-kind:Service","k8s-uid:a3b8afcc-9f19-42e4-aa8f-5866168c2ad3","k8s-group:","k8s-version:v1"],"host":"konghq.com","port":80}}]}`) expectedBody := map[string]interface{}{} require.NoError(t, json.Unmarshal(expectedRawErrBody, &expectedBody)) From 234d1aec5f27726e698f6fafb61e172a0bea7adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 29 May 2024 18:51:17 +0200 Subject: [PATCH 14/48] fix(gwapi): fix missing predicates for gateway-to-reconcile (#6097) * fix(gwapi): fix missing predicates for gateway-to-reconcile * Update test/envtest/specific_gateway_envtest_test.go --------- Co-authored-by: Jakub Warczarek --- .../configuration/kongupstreampolicy_utils.go | 6 +- internal/controllers/controller.go | 4 - .../controllers/gateway/gateway_controller.go | 87 ++++-- internal/controllers/gateway/gateway_utils.go | 7 +- .../controllers/gateway/gateway_utils_test.go | 271 ++++++++++++------ .../gateway/grpcroute_controller.go | 12 +- .../gateway/httproute_controller.go | 16 +- internal/controllers/gateway/route_utils.go | 5 +- .../controllers/gateway/route_utils_test.go | 16 +- .../gateway/tcproute_controller.go | 12 +- .../gateway/tlsroute_controller.go | 12 +- .../gateway/udproute_controller.go | 12 +- internal/controllers/namespacedname.go | 55 ++++ internal/controllers/namespacedname_test.go | 59 ++++ internal/manager/controllerdef.go | 12 +- test/envtest/gateway_envtest_test.go | 12 +- .../k8s_objects_status_envtest_test.go | 4 +- test/envtest/setup.go | 44 +-- test/envtest/specific_gateway_envtest_test.go | 237 +++++++++++++-- 19 files changed, 656 insertions(+), 227 deletions(-) create mode 100644 internal/controllers/namespacedname.go create mode 100644 internal/controllers/namespacedname_test.go diff --git a/internal/controllers/configuration/kongupstreampolicy_utils.go b/internal/controllers/configuration/kongupstreampolicy_utils.go index 7161765168..b29462973a 100644 --- a/internal/controllers/configuration/kongupstreampolicy_utils.go +++ b/internal/controllers/configuration/kongupstreampolicy_utils.go @@ -283,10 +283,8 @@ func getAllBackendRefsUsedWithService(httpRoute gatewayapi.HTTPRoute, serviceKey // We found a backendRef that matches the given service. We will keep all the backendRefs that are together // with this backendRef in the rule. - // Below we're suppressing nolintlint to not force `//nolint` instead of `// nolint`. This is to allow - // correctly suppressing looppointer which expects the latter. - backendRefs = append(backendRefs, rule.BackendRefs[:matchingIdx]...) // nolint:nolintlint,looppointer // We do not keep the reference to rule.BackendRefs, but copy it. - backendRefs = append(backendRefs, rule.BackendRefs[matchingIdx+1:]...) // nolint:nolintlint,looppointer // We do not keep the reference to rule.BackendRefs, but copy it. + backendRefs = append(backendRefs, rule.BackendRefs[:matchingIdx]...) // We do not keep the reference to rule.BackendRefs, but copy it. + backendRefs = append(backendRefs, rule.BackendRefs[matchingIdx+1:]...) // We do not keep the reference to rule.BackendRefs, but copy it. } } return backendRefs diff --git a/internal/controllers/controller.go b/internal/controllers/controller.go index 5e61b39b05..b1acf57911 100644 --- a/internal/controllers/controller.go +++ b/internal/controllers/controller.go @@ -2,8 +2,6 @@ package controllers import ( "github.com/go-logr/logr" - "github.com/samber/mo" - k8stypes "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -11,5 +9,3 @@ type Reconciler interface { SetupWithManager(ctrl.Manager) error SetLogger(logr.Logger) } - -type OptionalNamespacedName = mo.Option[k8stypes.NamespacedName] diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 67e54a4cf0..0972db0c02 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -170,10 +170,8 @@ func (r *GatewayReconciler) gatewayHasMatchingGatewayClass(obj client.Object) bo // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - return false - } + if !r.GatewayNN.Matches(gateway) { + return false } gatewayClass := &gatewayapi.GatewayClass{} @@ -203,6 +201,20 @@ func (r *GatewayReconciler) gatewayClassMatchesController(obj client.Object) boo // by a gatewayclass to enqueue them for reconciliation. This is generally used when a GatewayClass // is updated to ensure that idle gateways are initialized when their gatewayclass becomes available. func (r *GatewayReconciler) listGatewaysForGatewayClass(ctx context.Context, gatewayClass client.Object) []reconcile.Request { + // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. + // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 + if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { + gw := gatewayapi.Gateway{} + if err := r.Client.Get(ctx, gatewayToReconcile, &gw); err != nil { + r.Log.Error(err, "Failed to get gateways for gatewayclass in watch", + "gatewayclass", gatewayClass.GetName(), "gateway", gatewayToReconcile.String(), + ) + return nil + } + + return reconcileGatewaysIfClassMatches(gatewayClass, []gatewayapi.Gateway{gw}) + } + gateways := &gatewayapi.GatewayList{} if err := r.Client.List(ctx, gateways); err != nil { r.Log.Error(err, "Failed to list gateways for gatewayclass in watch", "gatewayclass", gatewayClass.GetName()) @@ -223,11 +235,26 @@ func (r *GatewayReconciler) listReferenceGrantsForGateway(ctx context.Context, o ) return nil } - gateways := &gatewayapi.GatewayList{} - if err := r.Client.List(ctx, gateways); err != nil { - r.Log.Error(err, "Failed to list gateways in watch", "referencegrant", grant.Name) - return nil + + var gateways gatewayapi.GatewayList + // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. + // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 + if gatewayToReconcileNN, ok := r.GatewayNN.Get(); ok { + gw := gatewayapi.Gateway{} + if err := r.Client.Get(ctx, gatewayToReconcileNN, &gw); err != nil { + r.Log.Error(err, "Failed to get gateway for referencegrant in watch", + "referencegrant", client.ObjectKeyFromObject(grant), "gateway", gatewayToReconcileNN.String(), + ) + return nil + } + gateways = gatewayapi.GatewayList{Items: []gatewayapi.Gateway{gw}} + } else { + if err := r.Client.List(ctx, &gateways); err != nil { + r.Log.Error(err, "Failed to list gateways in watch", "referencegrant", grant.Name) + return nil + } } + recs := []reconcile.Request{} for _, gateway := range gateways.Items { for _, from := range grant.Spec.From { @@ -251,16 +278,30 @@ func (r *GatewayReconciler) listReferenceGrantsForGateway(ctx context.Context, o // unmanaged mode and enqueues them for reconciliation. This is generally used to ensure // all gateways are updated when the service gets updated with new listeners. func (r *GatewayReconciler) listGatewaysForService(ctx context.Context, svc client.Object) (recs []reconcile.Request) { - gateways := &gatewayapi.GatewayList{} - if err := r.Client.List(ctx, gateways); err != nil { - r.Log.Error(err, "Failed to list gateways for service in watch predicates", "service", svc) - return + var gateways gatewayapi.GatewayList + // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. + // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 + if gatewayToReconcileNN, ok := r.GatewayNN.Get(); ok { + gw := gatewayapi.Gateway{} + if err := r.Client.Get(ctx, gatewayToReconcileNN, &gw); err != nil { + r.Log.Error(err, "Failed to get gateway for service in watch", + "service", client.ObjectKeyFromObject(svc), "gateway", gatewayToReconcileNN.String(), + ) + return nil + } + gateways = gatewayapi.GatewayList{Items: []gatewayapi.Gateway{gw}} + } else { + if err := r.Client.List(ctx, &gateways); err != nil { + r.Log.Error(err, "Failed to list gateways for service in watch", "service", svc.GetName()) + return nil + } } + for _, gateway := range gateways.Items { gatewayClass := &gatewayapi.GatewayClass{} if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: string(gateway.Spec.GatewayClassName)}, gatewayClass); err != nil { r.Log.Error(err, "Failed to retrieve gateway class in watch predicates", "gatewayclass", gateway.Spec.GatewayClassName) - return + return nil } if isGatewayClassControlled(gatewayClass) { recs = append(recs, reconcile.Request{ @@ -271,7 +312,7 @@ func (r *GatewayReconciler) listGatewaysForService(ctx context.Context, svc clie }) } } - return + return nil } // listGatewaysForHTTPRoute retrieves all the gateways referenced as parents by the HTTPRoute. @@ -286,7 +327,11 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(_ context.Context, obj clie return nil } recs := []reconcile.Request{} - for _, gateway := range routeAcceptedByGateways(httpRoute.Namespace, httpRoute.Status.Parents) { + for _, gateway := range routeAcceptedByGateways(httpRoute) { + if !r.GatewayNN.MatchesNN(gateway) { + continue + } + recs = append(recs, reconcile.Request{ NamespacedName: gateway, }) @@ -330,11 +375,13 @@ func referenceGrantHasGatewayFrom(obj client.Object) bool { func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("GatewayV1Gateway", req.NamespacedName) - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if req.Namespace != gatewayToReconcile.Namespace || req.Name != gatewayToReconcile.Name { - r.Log.V(util.DebugLevel).Info("The request does not match the specified Gateway and will be skipped.", "gateway", gatewayToReconcile.String()) - return ctrl.Result{}, nil - } + if nn, isSet := r.GatewayNN.Get(); isSet && !r.GatewayNN.MatchesNN(req.NamespacedName) { + r.Log.V(util.DebugLevel).Info( + "The request does not match the specified Gateway and will be skipped.", + "gateway", nn, + "request", req.String(), + ) + return ctrl.Result{}, nil } // gather the gateway object based on the reconciliation trigger. It's possible for the object diff --git a/internal/controllers/gateway/gateway_utils.go b/internal/controllers/gateway/gateway_utils.go index 270caabd6e..986eb5b458 100644 --- a/internal/controllers/gateway/gateway_utils.go +++ b/internal/controllers/gateway/gateway_utils.go @@ -664,10 +664,11 @@ func isTLSSecretValid(secret *corev1.Secret) bool { // routeAcceptedByGateways finds all the Gateways the route has been accepted by // and returns them in the form of a NamespacedName slice. -func routeAcceptedByGateways(routeNamespace string, parentStatuses []gatewayapi.RouteParentStatus) []k8stypes.NamespacedName { +func routeAcceptedByGateways(route *gatewayapi.HTTPRoute, +) []k8stypes.NamespacedName { gateways := []k8stypes.NamespacedName{} - for _, routeParentStatus := range parentStatuses { - gatewayNamespace := routeNamespace + for _, routeParentStatus := range getRouteStatusParents(route) { + gatewayNamespace := route.GetNamespace() parentRef := routeParentStatus.ParentRef if (parentRef.Group != nil && *parentRef.Group != gatewayapi.V1Group) || (parentRef.Kind != nil && *parentRef.Kind != "Gateway") { diff --git a/internal/controllers/gateway/gateway_utils_test.go b/internal/controllers/gateway/gateway_utils_test.go index 43a0ad91fc..e3b763828a 100644 --- a/internal/controllers/gateway/gateway_utils_test.go +++ b/internal/controllers/gateway/gateway_utils_test.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" @@ -291,23 +292,57 @@ func assertOnlyOneConditionForType(t *testing.T, conditions []metav1.Condition) func TestRouteAcceptedByGateways(t *testing.T) { testCases := []struct { - name string - routeNamespace string - parentStatuses []gatewayapi.RouteParentStatus - gateways []k8stypes.NamespacedName + name string + routeNamespace string + route *gatewayapi.HTTPRoute + gateways []client.Object + expectedGatewayNNs []k8stypes.NamespacedName }{ { - name: "no parentStatus with accepted condition", + name: "returns the gateway regardless of the route parent status conditions", routeNamespace: "default", - parentStatuses: []gatewayapi.RouteParentStatus{ - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-1", + route: &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "route-1", + }, + Status: gatewayapi.HTTPRouteStatus{ + RouteStatus: gatewayapi.RouteStatus{ + Parents: []gatewayapi.RouteParentStatus{ + { + ParentRef: gatewayapi.ParentReference{ + Name: "gateway-1", + }, + }, + }, }, }, }, - gateways: []k8stypes.NamespacedName{ - // Gateways should be included even when route is not accepted. + gateways: []client.Object{ + &gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gateway-1", + }, + Spec: gatewayapi.GatewaySpec{ + Listeners: builder.NewListener("http").WithPort(8080).HTTP().IntoSlice(), + }, + Status: gatewayapi.GatewayStatus{ + Listeners: []gatewayapi.ListenerStatus{ + { + Name: "http", + SupportedKinds: []gatewayapi.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayapi.V1Group), + Kind: gatewayapi.Kind("HTTPRoute"), + }, + }, + }, + }, + }, + }, + }, + expectedGatewayNNs: []k8stypes.NamespacedName{ { Namespace: "default", Name: "gateway-1", @@ -315,104 +350,172 @@ func TestRouteAcceptedByGateways(t *testing.T) { }, }, { - name: "a subset of parentStatus with correct params", + name: "returns the gateway regardless of the route parent status conditions", routeNamespace: "default", - parentStatuses: []gatewayapi.RouteParentStatus{ - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-1", - Group: lo.ToPtr(gatewayapi.Group("wrong-group")), - }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionTrue, - Type: string(gatewayapi.RouteConditionAccepted), + route: &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "route-1", + }, + Status: gatewayapi.HTTPRouteStatus{ + RouteStatus: gatewayapi.RouteStatus{ + Parents: []gatewayapi.RouteParentStatus{ + { + ParentRef: gatewayapi.ParentReference{ + Name: "gateway-1", + Group: lo.ToPtr(gatewayapi.Group("wrong-group")), + }, + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(gatewayapi.RouteConditionAccepted), + }, + }, + }, + { + ParentRef: gatewayapi.ParentReference{ + Name: "gateway-2", + Kind: lo.ToPtr(gatewayapi.Kind("wrong-kind")), + }, + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(gatewayapi.RouteConditionAccepted), + }, + }, + }, + { + ParentRef: gatewayapi.ParentReference{ + Name: "gateway-3", + }, + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(gatewayapi.RouteConditionAccepted), + }, + }, + }, + { + ParentRef: gatewayapi.ParentReference{ + Name: "gateway-4", + }, + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(gatewayapi.RouteConditionAccepted), + }, + }, + }, }, }, }, - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-2", - Kind: lo.ToPtr(gatewayapi.Kind("wrong-kind")), + }, + gateways: []client.Object{ + &gatewayapi.Gateway{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "wrong-group/v1", + Kind: "Gateway", }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionTrue, - Type: string(gatewayapi.RouteConditionAccepted), - }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gateway-1", }, - }, - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-3", + Spec: gatewayapi.GatewaySpec{ + Listeners: builder.NewListener("http").WithPort(8080).HTTP().IntoSlice(), }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionTrue, - Type: string(gatewayapi.RouteConditionAccepted), + Status: gatewayapi.GatewayStatus{ + Listeners: []gatewayapi.ListenerStatus{ + { + Name: "http", + SupportedKinds: []gatewayapi.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayapi.V1Group), + Kind: gatewayapi.Kind("HTTPRoute"), + }, + }, + }, }, }, }, - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-4", + &gatewayapi.Gateway{ + TypeMeta: metav1.TypeMeta{ + APIVersion: gatewayapi.GroupVersion.String(), + Kind: "wrong-kind", }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionFalse, - Type: string(gatewayapi.RouteConditionAccepted), + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gateway-2", + }, + Spec: gatewayapi.GatewaySpec{ + Listeners: builder.NewListener("http").WithPort(8080).HTTP().IntoSlice(), + }, + Status: gatewayapi.GatewayStatus{ + Listeners: []gatewayapi.ListenerStatus{ + { + Name: "http", + SupportedKinds: []gatewayapi.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayapi.V1Group), + Kind: gatewayapi.Kind("HTTPRoute"), + }, + }, + }, }, }, }, - }, - gateways: []k8stypes.NamespacedName{ - { - Namespace: "default", - Name: "gateway-3", - }, - // Gateways should be included even when route is not accepted. - { - Namespace: "default", - Name: "gateway-4", - }, - }, - }, - { - name: "all parentStatuses", - routeNamespace: "default", - parentStatuses: []gatewayapi.RouteParentStatus{ - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-1", + &gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gateway-3", }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionTrue, - Type: string(gatewayapi.RouteConditionAccepted), + Spec: gatewayapi.GatewaySpec{ + Listeners: builder.NewListener("http").WithPort(8080).HTTP().IntoSlice(), + }, + Status: gatewayapi.GatewayStatus{ + Listeners: []gatewayapi.ListenerStatus{ + { + Name: "http", + SupportedKinds: []gatewayapi.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayapi.V1Group), + Kind: gatewayapi.Kind("HTTPRoute"), + }, + }, + }, }, }, }, - { - ParentRef: gatewayapi.ParentReference{ - Name: "gateway-2", - Namespace: lo.ToPtr(gatewayapi.Namespace("namespace-2")), + &gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gateway-4", }, - Conditions: []metav1.Condition{ - { - Status: metav1.ConditionTrue, - Type: string(gatewayapi.RouteConditionAccepted), + Spec: gatewayapi.GatewaySpec{ + Listeners: builder.NewListener("http").WithPort(8080).HTTP().IntoSlice(), + }, + Status: gatewayapi.GatewayStatus{ + Listeners: []gatewayapi.ListenerStatus{ + { + Name: "http", + SupportedKinds: []gatewayapi.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayapi.V1Group), + Kind: gatewayapi.Kind("HTTPRoute"), + }, + }, + }, }, }, }, }, - gateways: []k8stypes.NamespacedName{ + expectedGatewayNNs: []k8stypes.NamespacedName{ { Namespace: "default", - Name: "gateway-1", + Name: "gateway-3", }, { - Namespace: "namespace-2", - Name: "gateway-2", + Namespace: "default", + Name: "gateway-4", }, }, }, @@ -420,8 +523,8 @@ func TestRouteAcceptedByGateways(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - gateways := routeAcceptedByGateways(tc.routeNamespace, tc.parentStatuses) - assert.Equal(t, tc.gateways, gateways) + gateways := routeAcceptedByGateways(tc.route) + assert.Equal(t, tc.expectedGatewayNNs, gateways) }) } } diff --git a/internal/controllers/gateway/grpcroute_controller.go b/internal/controllers/gateway/grpcroute_controller.go index 6a95ec5537..0b5cf7de7c 100644 --- a/internal/controllers/gateway/grpcroute_controller.go +++ b/internal/controllers/gateway/grpcroute_controller.go @@ -156,10 +156,8 @@ func (r *GRPCRouteReconciler) listGRPCRoutesForGatewayClass(ctx context.Context, if string(gateway.Spec.GatewayClassName) == gwc.Name { // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - continue - } + if !r.GatewayNN.Matches(&gateway) { + continue } _, ok := gateways[gateway.Namespace] @@ -238,10 +236,8 @@ func (r *GRPCRouteReconciler) listGRPCRoutesForGateway(ctx context.Context, obj // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gw.Namespace || gatewayToReconcile.Name != gw.Name { - return nil - } + if !r.GatewayNN.Matches(gw) { + return nil } // map all GRPCRoute objects diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index 454f6f1567..009a8715db 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -229,10 +229,8 @@ func (r *HTTPRouteReconciler) listHTTPRoutesForGatewayClass(ctx context.Context, if string(gateway.Spec.GatewayClassName) == gwc.Name { // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - continue - } + if !r.GatewayNN.Matches(&gateway) { + continue } _, ok := gateways[gateway.Namespace] @@ -311,10 +309,8 @@ func (r *HTTPRouteReconciler) listHTTPRoutesForGateway(ctx context.Context, obj // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gw.Namespace || gatewayToReconcile.Name != gw.Name { - return nil - } + if !r.GatewayNN.Matches(gw) { + return nil } // map all HTTPRoute objects @@ -418,7 +414,9 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Ensure we have no status for no-longer defined parentRefs. if wasAnyStatusRemoved := ensureNoStaleParentStatus(httproute); wasAnyStatusRemoved { if err := r.Status().Update(ctx, httproute); err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to prune stale Gateway parent statuses from %s status: %w", + client.ObjectKeyFromObject(httproute), err, + ) } return ctrl.Result{}, nil } diff --git a/internal/controllers/gateway/route_utils.go b/internal/controllers/gateway/route_utils.go index df6a921aea..cb42bf58ff 100644 --- a/internal/controllers/gateway/route_utils.go +++ b/internal/controllers/gateway/route_utils.go @@ -127,7 +127,9 @@ func parentRefsForRoute[T gatewayapi.RouteT](route T) ([]gatewayapi.ParentRefere // OR the present gateways are references to missing objects, this will return a unsupportedGW error. // // There is a parameter `specifiedGW` here, which is used to specific the gateway. -func getSupportedGatewayForRoute[T gatewayapi.RouteT](ctx context.Context, logger logr.Logger, mgrc client.Client, route T, specifiedGW controllers.OptionalNamespacedName) ([]supportedGatewayWithCondition, error) { +func getSupportedGatewayForRoute[T gatewayapi.RouteT]( + ctx context.Context, logger logr.Logger, mgrc client.Client, route T, specifiedGW controllers.OptionalNamespacedName, +) ([]supportedGatewayWithCondition, error) { // gather the parentrefs for this route object parentRefs, err := parentRefsForRoute(route) if err != nil { @@ -1019,7 +1021,6 @@ func ensureGatewayReferenceStatusRemoved[routeT gatewayapi.RouteT]( // it it's possible it became orphaned after becoming queued. In either case // ensure that it's removed from the proxy cache to avoid orphaned data-plane // configurations. - debug(log, route, "Ensuring that dataplane is updated to remove unsupported route (if applicable)") setRouteStatusParents(route, newStatuses) if err := cl.Status().Update(ctx, route); err != nil { return false, fmt.Errorf("failed to remove Gateway parentRef from %s status: %w", kind, err) diff --git a/internal/controllers/gateway/route_utils_test.go b/internal/controllers/gateway/route_utils_test.go index e61843e4e0..4febd8a2b4 100644 --- a/internal/controllers/gateway/route_utils_test.go +++ b/internal/controllers/gateway/route_utils_test.go @@ -582,7 +582,7 @@ func TestGetSupportedGatewayForRoute(t *testing.T) { WithObjects(tt.objects...). Build() - got, err := getSupportedGatewayForRoute(context.Background(), logr.Discard(), fakeClient, tt.route, controllers.OptionalNamespacedName{}) + got, err := getSupportedGatewayForRoute(context.Background(), logr.Discard(), fakeClient, tt.route, controllers.NewOptionalNamespacedName(mo.None[k8stypes.NamespacedName]())) require.NoError(t, err) require.Len(t, got, len(tt.expected)) @@ -1425,7 +1425,7 @@ func TestGetSupportedGatewayForRoute(t *testing.T) { }, { name: "HTTPRoute with other parent Gateway finds no matching Gateways", - route: basicHTTPRoute("bad-gateway"), + route: basicHTTPRoute("good-gateway"), objects: []client.Object{ namedGateway("bad-gateway"), gatewayClass, @@ -1450,10 +1450,14 @@ func TestGetSupportedGatewayForRoute(t *testing.T) { logr.Discard(), fakeClient, tt.route, - mo.Some(k8stypes.NamespacedName{ - Namespace: namespace.Name, - Name: "good-gateway", - }), + controllers.NewOptionalNamespacedName( + mo.Some( + k8stypes.NamespacedName{ + Namespace: namespace.Name, + Name: "good-gateway", + }, + ), + ), ) if tt.expectedErr != nil { require.Equal(t, tt.expectedErr, err) diff --git a/internal/controllers/gateway/tcproute_controller.go b/internal/controllers/gateway/tcproute_controller.go index 03ae617628..1f135e7751 100644 --- a/internal/controllers/gateway/tcproute_controller.go +++ b/internal/controllers/gateway/tcproute_controller.go @@ -152,10 +152,8 @@ func (r *TCPRouteReconciler) listTCPRoutesForGatewayClass(ctx context.Context, o if string(gateway.Spec.GatewayClassName) == gwc.Name { // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - continue - } + if !r.GatewayNN.Matches(&gateway) { + continue } _, ok := gateways[gateway.Namespace] @@ -234,10 +232,8 @@ func (r *TCPRouteReconciler) listTCPRoutesForGateway(ctx context.Context, obj cl // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gw.Namespace || gatewayToReconcile.Name != gw.Name { - return nil - } + if !r.GatewayNN.Matches(gw) { + return nil } // map all TCPRoute objects diff --git a/internal/controllers/gateway/tlsroute_controller.go b/internal/controllers/gateway/tlsroute_controller.go index 6b64026e62..96de3a6282 100644 --- a/internal/controllers/gateway/tlsroute_controller.go +++ b/internal/controllers/gateway/tlsroute_controller.go @@ -147,10 +147,8 @@ func (r *TLSRouteReconciler) listTLSRoutesForGatewayClass(ctx context.Context, o if string(gateway.Spec.GatewayClassName) == gwc.Name { // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - continue - } + if !r.GatewayNN.Matches(&gateway) { + continue } _, ok := gateways[gateway.Namespace] @@ -229,10 +227,8 @@ func (r *TLSRouteReconciler) listTLSRoutesForGateway(ctx context.Context, obj cl // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gw.Namespace || gatewayToReconcile.Name != gw.Name { - return nil - } + if !r.GatewayNN.Matches(gw) { + return nil } // map all TLSRoute objects diff --git a/internal/controllers/gateway/udproute_controller.go b/internal/controllers/gateway/udproute_controller.go index 09e52c3e4a..aa8bb52c6e 100644 --- a/internal/controllers/gateway/udproute_controller.go +++ b/internal/controllers/gateway/udproute_controller.go @@ -151,10 +151,8 @@ func (r *UDPRouteReconciler) listUDPRoutesForGatewayClass(ctx context.Context, o if string(gateway.Spec.GatewayClassName) == gwc.Name { // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gateway.Namespace || gatewayToReconcile.Name != gateway.Name { - continue - } + if !r.GatewayNN.Matches(&gateway) { + continue } _, ok := gateways[gateway.Namespace] @@ -233,10 +231,8 @@ func (r *UDPRouteReconciler) listUDPRoutesForGateway(ctx context.Context, obj cl // If the flag `--gateway-to-reconcile` is set, KIC will only reconcile the specified gateway. // https://github.com/Kong/kubernetes-ingress-controller/issues/5322 - if gatewayToReconcile, ok := r.GatewayNN.Get(); ok { - if gatewayToReconcile.Namespace != gw.Namespace || gatewayToReconcile.Name != gw.Name { - return nil - } + if !r.GatewayNN.Matches(gw) { + return nil } // map all UDPRoute objects diff --git a/internal/controllers/namespacedname.go b/internal/controllers/namespacedname.go new file mode 100644 index 0000000000..ac0d7dbcb0 --- /dev/null +++ b/internal/controllers/namespacedname.go @@ -0,0 +1,55 @@ +package controllers + +import ( + "github.com/samber/mo" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// OptionalNamespacedName is a wrapper around mo.Option[k8stypes.NamespacedName] that provides +// additional Matches and MatchesNN methods for matching against client.Object and +// k8stypes.NamespacedName. +type OptionalNamespacedName struct { + mo.Option[k8stypes.NamespacedName] +} + +// NewOptionalNamespacedName creates a new OptionalNamespacedName with the provided value. +func NewOptionalNamespacedName(onn mo.Option[k8stypes.NamespacedName]) OptionalNamespacedName { + return OptionalNamespacedName{onn} +} + +// Get calls the underlying mo.Option.Get. +func (onn OptionalNamespacedName) Get() (k8stypes.NamespacedName, bool) { + return onn.Option.Get() +} + +// IsPresent calls the underlying mo.Option.IsPresent. +func (onn OptionalNamespacedName) IsPresent() bool { + return onn.Option.IsPresent() +} + +// Matches returns true if the OptionalNamespacedName is present and the provided object's +// namespace and name match the OptionalNamespacedName's namespace and name. +// It also returns true if the OptionalNamespacedName is not present as it is considered +// to match everything. +func (onn OptionalNamespacedName) Matches(obj client.Object) bool { + n, ok := onn.Option.Get() + if !ok { + return true + } + + return n.Namespace == obj.GetNamespace() && n.Name == obj.GetName() +} + +// MatchesNN returns true if the OptionalNamespacedName is present and the provided +// k8stypes.NamespacedName matches the OptionalNamespacedName's namespace and name. +// It also returns true if the OptionalNamespacedName is not present as it is considered +// to match everything. +func (onn OptionalNamespacedName) MatchesNN(nn k8stypes.NamespacedName) bool { + n, ok := onn.Option.Get() + if !ok { + return true + } + + return n.Namespace == nn.Namespace && n.Name == nn.Name +} diff --git a/internal/controllers/namespacedname_test.go b/internal/controllers/namespacedname_test.go new file mode 100644 index 0000000000..51ddfa8012 --- /dev/null +++ b/internal/controllers/namespacedname_test.go @@ -0,0 +1,59 @@ +package controllers + +import ( + "testing" + + "github.com/samber/mo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + + "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" +) + +func TestOptionalNamespacedName(t *testing.T) { + t.Run("Get empty", func(t *testing.T) { + emptyOptional := OptionalNamespacedName{} + nn, ok := emptyOptional.Get() + require.False(t, ok) + assert.Equal(t, k8stypes.NamespacedName{}, nn) + }) + + t.Run("Get nonempty", func(t *testing.T) { + namespacedName := k8stypes.NamespacedName{ + Name: "example", + Namespace: "default", + } + presentOptional := NewOptionalNamespacedName(mo.Some(namespacedName)) + nn, ok := presentOptional.Get() + require.True(t, ok) + assert.Equal(t, namespacedName, nn) + }) + + t.Run("Matches", func(t *testing.T) { + namespacedName := k8stypes.NamespacedName{ + Name: "example", + Namespace: "default", + } + presentOptional := NewOptionalNamespacedName(mo.Some(namespacedName)) + assert.True(t, presentOptional.Matches(&gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "default", + }, + })) + assert.False(t, presentOptional.Matches(&gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy", + Namespace: "default", + }, + })) + assert.False(t, presentOptional.Matches(&gatewayapi.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: "dummy", + }, + })) + }) +} diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 4918469b07..5f5bdd68d1 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -318,7 +318,7 @@ func setupControllers( WatchNamespaces: c.WatchNamespaces, CacheSyncTimeout: c.CacheSyncTimeout, ReferenceIndexers: referenceIndexers, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, @@ -340,7 +340,7 @@ func setupControllers( DataplaneClient: dataplaneClient, CacheSyncTimeout: c.CacheSyncTimeout, StatusQueue: kubernetesStatusQueue, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, @@ -382,7 +382,7 @@ func setupControllers( DataplaneClient: dataplaneClient, CacheSyncTimeout: c.CacheSyncTimeout, StatusQueue: kubernetesStatusQueue, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, @@ -407,7 +407,7 @@ func setupControllers( DataplaneClient: dataplaneClient, CacheSyncTimeout: c.CacheSyncTimeout, StatusQueue: kubernetesStatusQueue, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, @@ -429,7 +429,7 @@ func setupControllers( DataplaneClient: dataplaneClient, CacheSyncTimeout: c.CacheSyncTimeout, StatusQueue: kubernetesStatusQueue, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, @@ -451,7 +451,7 @@ func setupControllers( DataplaneClient: dataplaneClient, CacheSyncTimeout: c.CacheSyncTimeout, StatusQueue: kubernetesStatusQueue, - GatewayNN: c.GatewayToReconcile, + GatewayNN: controllers.NewOptionalNamespacedName(c.GatewayToReconcile), }, }, }, diff --git a/test/envtest/gateway_envtest_test.go b/test/envtest/gateway_envtest_test.go index f76dc74869..d0ba9f070b 100644 --- a/test/envtest/gateway_envtest_test.go +++ b/test/envtest/gateway_envtest_test.go @@ -32,7 +32,7 @@ func TestGatewayAddressOverride(t *testing.T) { defer cancel() expected := []string{"10.0.0.1", "10.0.0.2"} udp := []string{"10.0.0.3", "10.0.0.4"} - gw := deployGateway(ctx, t, ctrlClient) + gw, _ := deployGateway(ctx, t, ctrlClient) RunManager(ctx, t, envcfg, AdminAPIOptFns(), WithPublishService(gw.Namespace), @@ -80,7 +80,7 @@ func TestGatewayReconciliation_MoreThan100Routes(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - gw := deployGateway(ctx, t, ctrlClient) + gw, _ := deployGateway(ctx, t, ctrlClient) RunManager(ctx, t, envcfg, AdminAPIOptFns(), WithPublishService(gw.Namespace), @@ -119,7 +119,7 @@ func createHTTPRoutes( ctrlClient ctrlclient.Client, gw gatewayapi.Gateway, numOfRoutes int, -) { +) []*gatewayapi.HTTPRoute { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "backend-svc", @@ -138,6 +138,7 @@ func createHTTPRoutes( require.NoError(t, ctrlClient.Create(ctx, svc)) t.Cleanup(func() { _ = ctrlClient.Delete(ctx, svc) }) + routes := make([]*gatewayapi.HTTPRoute, 0, numOfRoutes) for i := 0; i < numOfRoutes; i++ { httpPort := gatewayapi.PortNumber(80) pathMatchPrefix := gatewayapi.PathMatchPathPrefix @@ -174,8 +175,9 @@ func createHTTPRoutes( }, } - err := ctrlClient.Create(ctx, httpRoute) - require.NoError(t, err) + require.NoError(t, ctrlClient.Create(ctx, httpRoute)) t.Cleanup(func() { _ = ctrlClient.Delete(ctx, httpRoute) }) + routes = append(routes, httpRoute) } + return routes } diff --git a/test/envtest/k8s_objects_status_envtest_test.go b/test/envtest/k8s_objects_status_envtest_test.go index 6c36d711df..a3f1eca106 100644 --- a/test/envtest/k8s_objects_status_envtest_test.go +++ b/test/envtest/k8s_objects_status_envtest_test.go @@ -28,7 +28,7 @@ func TestHTTPRouteReconciliation_DoesNotBlockSyncLoopWhenStatusQueueBufferIsExce ctx, cancel := context.WithCancel(context.Background()) defer cancel() - gw := deployGateway(ctx, t, ctrlClient) + gw, _ := deployGateway(ctx, t, ctrlClient) RunManager(ctx, t, envcfg, AdminAPIOptFns(), WithPublishService(gw.Namespace), @@ -127,7 +127,7 @@ func Test_WatchNamespaces(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - gw := deployGateway(ctx, t, ctrlClient) + gw, _ := deployGateway(ctx, t, ctrlClient) hidden := CreateNamespace(ctx, t, ctrlClient) RunManager(ctx, t, envcfg, AdminAPIOptFns(), diff --git a/test/envtest/setup.go b/test/envtest/setup.go index 9c5160b30a..d0432e806e 100644 --- a/test/envtest/setup.go +++ b/test/envtest/setup.go @@ -169,7 +169,7 @@ func deployIngressClass(ctx context.Context, t *testing.T, name string, client c } // deployGateway deploys a Gateway, GatewayClass, and ingress service for use in tests. -func deployGateway(ctx context.Context, t *testing.T, client ctrlclient.Client) gatewayapi.Gateway { +func deployGatewayUsingGatewayClass(ctx context.Context, t *testing.T, client ctrlclient.Client, gwc gatewayapi.GatewayClass) gatewayapi.Gateway { ns := CreateNamespace(ctx, t, client) publishSvc := corev1.Service{ @@ -188,29 +188,15 @@ func deployGateway(ctx context.Context, t *testing.T, client ctrlclient.Client) require.NoError(t, client.Create(ctx, &publishSvc)) t.Cleanup(func() { _ = client.Delete(ctx, &publishSvc) }) - gwc := gatewayapi.GatewayClass{ - Spec: gatewayapi.GatewayClassSpec{ - ControllerName: gateway.GetControllerName(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), - Annotations: map[string]string{ - "konghq.com/gatewayclass-unmanaged": "placeholder", - }, - }, - } - - require.NoError(t, client.Create(ctx, &gwc)) - t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) - gw := gatewayapi.Gateway{ Spec: gatewayapi.GatewaySpec{ GatewayClassName: gatewayapi.ObjectName(gwc.Name), Listeners: []gatewayapi.Listener{ { - Name: "http", - Protocol: gatewayapi.HTTPProtocolType, - Port: gatewayapi.PortNumber(8000), + Name: "http", + Protocol: gatewayapi.HTTPProtocolType, + Port: gatewayapi.PortNumber(8000), + AllowedRoutes: builder.NewAllowedRoutesFromAllNamespaces(), }, }, }, @@ -224,3 +210,23 @@ func deployGateway(ctx context.Context, t *testing.T, client ctrlclient.Client) return gw } + +func deployGateway(ctx context.Context, t *testing.T, client ctrlclient.Client) (gatewayapi.Gateway, gatewayapi.GatewayClass) { + gwc := gatewayapi.GatewayClass{ + Spec: gatewayapi.GatewayClassSpec{ + ControllerName: gateway.GetControllerName(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{ + "konghq.com/gatewayclass-unmanaged": "placeholder", + }, + }, + } + require.NoError(t, client.Create(ctx, &gwc)) + t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) + + gw := deployGatewayUsingGatewayClass(ctx, t, client, gwc) + + return gw, gwc +} diff --git a/test/envtest/specific_gateway_envtest_test.go b/test/envtest/specific_gateway_envtest_test.go index 432da1f111..95e021ec98 100644 --- a/test/envtest/specific_gateway_envtest_test.go +++ b/test/envtest/specific_gateway_envtest_test.go @@ -4,13 +4,14 @@ package envtest import ( "context" - "fmt" + "strconv" "testing" "time" "github.com/samber/lo" "github.com/stretchr/testify/require" - k8stypes "k8s.io/apimachinery/pkg/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" ) @@ -19,7 +20,7 @@ func TestSpecificGatewayNN(t *testing.T) { t.Parallel() const ( - waitTime = time.Minute + waitTime = 3 * time.Second tickTime = 500 * time.Millisecond ) @@ -29,49 +30,223 @@ func TestSpecificGatewayNN(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - gw := deployGateway(ctx, t, ctrlClient) - ignoredGW := deployGateway(ctx, t, ctrlClient) + + var ( + gw, gwc = deployGateway(ctx, t, ctrlClient) + nn = client.ObjectKeyFromObject(&gw) + // We use the same GatewayClass here. + gwIgnored = deployGatewayUsingGatewayClass(ctx, t, ctrlClient, gwc) + nnIgnored = client.ObjectKeyFromObject(&gwIgnored) + ) + RunManager(ctx, t, envcfg, AdminAPIOptFns(), WithPublishService(gw.Namespace), WithGatewayFeatureEnabled, WithGatewayAPIControllers(), - WithGatewayToReconcile(fmt.Sprintf("%s/%s", gw.Namespace, gw.Name)), + WithGatewayToReconcile(nn.String()), ) - createHTTPRoutes(ctx, t, ctrlClient, gw, 1) - createHTTPRoutes(ctx, t, ctrlClient, ignoredGW, 1) + const routeCount = 10 + routes := createHTTPRoutes(ctx, t, ctrlClient, gw, routeCount) + ignoredRoutes := createHTTPRoutes(ctx, t, ctrlClient, gwIgnored, routeCount) + + t.Run("configured specific gateway gets its listener status filled", func(t *testing.T) { + require.Eventually(t, func() bool { + if err := ctrlClient.Get(ctx, nn, &gw); err != nil { + t.Logf("Failed to get gateway %s: %v", nn, err) + return false + } + httpListener, ok := lo.Find(gw.Status.Listeners, func(listener gatewayapi.ListenerStatus) bool { + return listener.Name == "http" + }) + if !ok { + t.Logf("Failed to find http listener status in gateway %s", nn) + return false + } + if httpListener.AttachedRoutes != int32(routeCount) { + t.Logf("Expected %d routes to be attached to the http listener, got %d", routeCount, httpListener.AttachedRoutes) + return false + } + + return true + }, waitTime, tickTime, "Failed to attach route to gateway") + }) + t.Run("HTTPRoute attached to configured specific gateway gets its status parent filled", func(t *testing.T) { + require.Eventually(t, func() bool { + route := gatewayapi.HTTPRoute{} + routeNN := client.ObjectKeyFromObject(routes[0]) + if err := ctrlClient.Get(ctx, routeNN, &route); err != nil { + t.Logf("Failed to get route %s: %v", routeNN, err) + return false + } + for _, p := range route.Status.Parents { + return lo.ContainsBy(p.Conditions, func(c metav1.Condition) bool { + return c.Type == string(gatewayapi.RouteConditionAccepted) && c.Status == metav1.ConditionTrue + }) + } - require.Eventually(t, func() bool { - err := ctrlClient.Get(ctx, k8stypes.NamespacedName{Namespace: gw.Namespace, Name: gw.Name}, &gw) - if err != nil { - t.Logf("Failed to get gateway %s/%s: %v", gw.Namespace, gw.Name, err) return false - } - httpListener, ok := lo.Find(gw.Status.Listeners, func(listener gatewayapi.ListenerStatus) bool { - return listener.Name == "http" - }) - if !ok { - t.Logf("Failed to find http listener status in gateway %s/%s", gw.Namespace, gw.Name) + }, waitTime, tickTime, "Failed to get accepted condition on HTTPRoute") + }) + + t.Run("not configured gateway does not gets its listener status filled and HTTPRoute attached to it doesn't get its status parent filled", func(t *testing.T) { + require.Never(t, func() bool { + t.Logf("Checking if Gateway %s is ignored (does not get status listeners filled)", nnIgnored) + if err := ctrlClient.Get(ctx, nnIgnored, &gwIgnored); err != nil { + t.Logf("Failed to get gateway %s: %v", nnIgnored, err) + return true + } + + if len(gwIgnored.Status.Listeners) != 0 { + t.Logf("%s gateway should not have status listeners filled.", nnIgnored) + return true + } + // Programmed and Accepted conditions are set by default to Unknown. + if gatewayStatusContainsProgrammedOrAcceptedCondition(t, gwIgnored) { + return true + } + + t.Logf("Checking if HTTPRoute %s is ignored as expected", ignoredRoutes[0].Name) + routeIgnored := gatewayapi.HTTPRoute{} + routeIgnoredNN := client.ObjectKeyFromObject(ignoredRoutes[0]) + if err := ctrlClient.Get(ctx, routeIgnoredNN, &routeIgnored); err != nil { + t.Logf("Failed to get ignored route %s: %v", routeIgnoredNN, err) + return false + } + for _, p := range routeIgnored.Status.Parents { + return lo.ContainsBy(p.Conditions, func(c metav1.Condition) bool { + return c.Type == string(gatewayapi.RouteConditionAccepted) && + c.Status == metav1.ConditionTrue + }) + } + return false - } - if httpListener.AttachedRoutes != 1 { - t.Logf("Expected %d routes to be attached to the http listener, got %d", 1, httpListener.AttachedRoutes) + }, waitTime, tickTime, "Non configured gateway should not be processed and HTTPRoute should not be accepted") + }) + + t.Run("changes to gatewayclass used by not configured gateway do not get gateway's listener status filled", func(t *testing.T) { + require.Never(t, func() bool { + gwcOld := gwc.DeepCopy() + gwc.Annotations = map[string]string{"foo": strconv.Itoa(time.Now().Nanosecond())} + if err := ctrlClient.Patch(ctx, &gwc, client.MergeFrom(gwcOld)); err != nil { + t.Logf("Failed patching gatewayclass %s: %v", client.ObjectKeyFromObject(&gwc), err) + return true + } + + t.Logf("Checking if Gateway %s is ignored (does not get status listeners filled)", nnIgnored) + if err := ctrlClient.Get(ctx, nnIgnored, &gwIgnored); err != nil { + t.Logf("Failed to get gateway %s: %v", nnIgnored, err) + return true + } + + if len(gwIgnored.Status.Listeners) != 0 { + t.Logf("%s gateway should not be processed.", nnIgnored) + return true + } + + // Programmed and Accepted conditions are set by default to Unknown. + if gatewayStatusContainsProgrammedOrAcceptedCondition(t, gwIgnored) { + return true + } + return false - } + }, waitTime, tickTime) + }) + + t.Run("changes to httproute attached to ignored gateway do not get gateway's listener status filled", func(t *testing.T) { + require.Never(t, func() bool { + routeIgnored := ignoredRoutes[0].DeepCopy() + routeIgnoredOld := routeIgnored.DeepCopy() + routeIgnored.Annotations = map[string]string{"foo": strconv.Itoa(time.Now().Nanosecond())} + if err := ctrlClient.Patch(ctx, routeIgnored, client.MergeFrom(routeIgnoredOld)); err != nil { + t.Logf("Failed patching gatewayclass %s: %v", client.ObjectKeyFromObject(routeIgnored), err) + return true + } + + t.Logf("Checking if Gateway %s is ignored (does not get status listeners filled)", gwIgnored.Name) + if err := ctrlClient.Get(ctx, nnIgnored, &gwIgnored); err != nil { + t.Logf("Failed to get gateway %s: %v", nnIgnored, err) + return true + } - err = ctrlClient.Get(ctx, k8stypes.NamespacedName{Namespace: ignoredGW.Namespace, Name: ignoredGW.Name}, &ignoredGW) - if err != nil { - t.Logf("Failed to get gateway %s/%s: %v", ignoredGW.Namespace, ignoredGW.Name, err) + if len(gwIgnored.Status.Listeners) != 0 { + t.Logf("%s gateway should not have status listeners filled.", nnIgnored) + return true + } + + // Programmed and Accepted conditions are set by default to Unknown. + if gatewayStatusContainsProgrammedOrAcceptedCondition(t, gwIgnored) { + return true + } return false + }, waitTime, tickTime) + }) + + t.Run("changes to referencegrant used by not configured gateway do not get gateway's listener status filled", func(t *testing.T) { + refGrant := gatewayapi.ReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "refgrant-1", + Namespace: gwIgnored.Namespace, + }, + Spec: gatewayapi.ReferenceGrantSpec{ + From: []gatewayapi.ReferenceGrantFrom{ + { + Group: gatewayapi.V1Group, + Kind: "Gateway", + Namespace: gatewayapi.Namespace(gwIgnored.Namespace), + }, + }, + To: []gatewayapi.ReferenceGrantTo{ + { + Group: gatewayapi.V1Group, + Kind: "HTTPRoute", + }, + }, + }, } + require.NoError(t, ctrlClient.Create(ctx, &refGrant)) + + require.Never(t, func() bool { + refGrantOld := refGrant.DeepCopy() + refGrant.Annotations = map[string]string{"foo": strconv.Itoa(time.Now().Nanosecond())} + if err := ctrlClient.Patch(ctx, &refGrant, client.MergeFrom(refGrantOld)); err != nil { + t.Logf("Failed patching referenceGrant %s: %v", client.ObjectKeyFromObject(&refGrant), err) + return true + } + + t.Logf("Checking if Gateway %s is ignored (does not get status listeners filled)", nnIgnored) + if err := ctrlClient.Get(ctx, nnIgnored, &gwIgnored); err != nil { + t.Logf("Failed to get gateway %s: %v", nnIgnored, err) + return true + } + + if len(gwIgnored.Status.Listeners) != 0 { + t.Logf("%s gateway should not be processed.", nnIgnored) + return true + } + + // Programmed and Accepted conditions are set by default to Unknown. + if gatewayStatusContainsProgrammedOrAcceptedCondition(t, gwIgnored) { + return true + } - // ignoredGW.Status.Listeners should be [] - if len(ignoredGW.Status.Listeners) != 0 { - t.Logf("Expected %s/%s gateway should not be processed.", ignoredGW.Namespace, ignoredGW.Name) return false - } + }, waitTime, tickTime) + }) +} - return true - }, waitTime, tickTime, "failed to reconcile all HTTPRoutes") +func gatewayStatusContainsProgrammedOrAcceptedCondition(t *testing.T, gw gatewayapi.Gateway) bool { + nn := client.ObjectKeyFromObject(&gw) + for _, c := range gw.Status.Conditions { + if c.Type == string(gatewayapi.GatewayConditionProgrammed) && c.Status != metav1.ConditionUnknown { + t.Logf("%s gateway should not have Programmed status condition set to something else than Unknown.", nn) + return true + } + if c.Type == string(gatewayapi.GatewayConditionAccepted) && c.Status != metav1.ConditionUnknown { + t.Logf("%s gateway should not have Accepted status condition set to something else than Unknown.", nn) + return true + } + } + return false } From aaf22bff01219d225739d31b00ca42185e869440 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 16:52:01 +0000 Subject: [PATCH 15/48] chore(deps): update kong/kong-gateway docker tag to v3.7.0.0 (#6102) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/test_dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/test_dependencies.yaml b/.github/test_dependencies.yaml index dc440dd7da..6614ee4a1f 100644 --- a/.github/test_dependencies.yaml +++ b/.github/test_dependencies.yaml @@ -46,7 +46,7 @@ integration: # renovate: datasource=docker depName=kong versioning=docker kong-oss: '3.6.1' # renovate: datasource=docker depName=kong/kong-gateway versioning=docker - kong-ee: '3.6.1.4' + kong-ee: '3.7.0.0' kongintegration: # renovate: datasource=docker depName=kong versioning=docker From 59aeee6078ec80667ae34be00031032e44e1e5b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 16:53:40 +0000 Subject: [PATCH 16/48] chore(deps): bump google.golang.org/api from 0.181.0 to 0.182.0 (#6106) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.181.0 to 0.182.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.181.0...v0.182.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index a80bf3eed6..950f4c72e2 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 go.uber.org/zap v1.27.0 - google.golang.org/api v0.181.0 + google.golang.org/api v0.182.0 k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.30.1 k8s.io/apimachinery v0.30.1 @@ -65,7 +65,7 @@ require ( ) require ( - cloud.google.com/go/auth v0.4.1 // indirect + cloud.google.com/go/auth v0.4.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect @@ -214,8 +214,8 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index 44da2e84c6..ba3e37e8ea 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= -cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= -cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= -cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= +cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= @@ -583,17 +583,17 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4= -google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k= +google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= +google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= From 7056cbc358bfe6f9bfcbe93552e4ab5a790bf202 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 15:02:15 +0800 Subject: [PATCH 17/48] chore(deps): update dependency gotestyourself/gotestsum to v1.12.0 (#6109) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .tools_versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tools_versions.yaml b/.tools_versions.yaml index 2b867be195..a0b56dd1e8 100644 --- a/.tools_versions.yaml +++ b/.tools_versions.yaml @@ -17,6 +17,6 @@ yq: "4.44.1" # renovate: datasource=github-releases depName=jstemmer/go-junit-report gojunit-report: "2.1.0" # renovate: datasource=github-releases depName=gotestyourself/gotestsum -gotestsum: "1.11.0" +gotestsum: "1.12.0" # renovate: datasource=github-releases depName=dominikh/go-tools staticcheck: "0.4.7" From 1d14aa04da959bb533e326958a2a45620d3da871 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 20:36:53 +0200 Subject: [PATCH 18/48] chore(deps): bump github.com/hashicorp/go-retryablehttp (#6107) Bumps [github.com/hashicorp/go-retryablehttp](https://github.com/hashicorp/go-retryablehttp) from 0.7.6 to 0.7.7. - [Changelog](https://github.com/hashicorp/go-retryablehttp/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/go-retryablehttp/compare/v0.7.6...v0.7.7) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-retryablehttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 950f4c72e2..cb932e6283 100644 --- a/go.mod +++ b/go.mod @@ -132,7 +132,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.4 // indirect - github.com/hashicorp/go-retryablehttp v0.7.6 + github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/huandu/xstrings v1.4.0 // indirect diff --git a/go.sum b/go.sum index ba3e37e8ea..a029648e1f 100644 --- a/go.sum +++ b/go.sum @@ -214,8 +214,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= -github.com/hashicorp/go-retryablehttp v0.7.6 h1:TwRYfx2z2C4cLbXmT8I5PgP/xmuqASDyiVuGYfs9GZM= -github.com/hashicorp/go-retryablehttp v0.7.6/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= From 8b836879270079bc801266fed085cb5f9a8c643b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Fri, 31 May 2024 21:20:24 +0200 Subject: [PATCH 19/48] deps: bump sigs.k8s.io/e2e-framework to v0.4.0 (#6113) --- go.mod | 6 ++---- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cb932e6283..9a4337ad63 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/kong/kubernetes-ingress-controller/v3 -go 1.22.0 - -toolchain go1.22.2 +go 1.22.3 // TODO: this is disabled by FOSSA action doesn't support go 1.21's toolchain clause: // @@ -58,6 +56,7 @@ require ( k8s.io/client-go v0.30.1 k8s.io/component-base v0.30.1 sigs.k8s.io/controller-runtime v0.18.3 + sigs.k8s.io/e2e-framework v0.4.0 sigs.k8s.io/gateway-api v1.1.0 sigs.k8s.io/kustomize/api v0.17.2 sigs.k8s.io/kustomize/kyaml v0.17.1 @@ -227,7 +226,6 @@ require ( k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect k8s.io/kubectl v0.30.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect - sigs.k8s.io/e2e-framework v0.3.1-0.20231113122213-262cac32d35e sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kind v0.22.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index a029648e1f..065f4a85b3 100644 --- a/go.sum +++ b/go.sum @@ -659,8 +659,8 @@ k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCI k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.18.3 h1:B5Wmmo8WMWK7izei+2LlXLVDGzMwAHBNLX68lwtlSR4= sigs.k8s.io/controller-runtime v0.18.3/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= -sigs.k8s.io/e2e-framework v0.3.1-0.20231113122213-262cac32d35e h1:lJqSZb2bAyfkPpBhUbzXsoAHKJn+3/KzBpvktR1wlMQ= -sigs.k8s.io/e2e-framework v0.3.1-0.20231113122213-262cac32d35e/go.mod h1:VIozg+of0zhkVGZcCWvKqH7vn7GTDDAa3h+w1OfH7Co= +sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY= +sigs.k8s.io/e2e-framework v0.4.0/go.mod h1:JilFQPF1OL1728ABhMlf9huse7h+uBJDXl9YeTs49A8= sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 3abda09558122291d5eabca3f08481050ce4d864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Fri, 31 May 2024 21:21:14 +0200 Subject: [PATCH 20/48] conformance: enable HTTPRoutePathRewrite test (#6114) --- test/conformance/gateway_conformance_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 014bf6b3a6..1df08ba212 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -32,7 +32,6 @@ var traditionalRoutesSupportedFeatures = []features.SupportedFeature{ features.SupportHTTPRoute, // extended features features.SupportHTTPRouteResponseHeaderModification, - features.SupportHTTPRoutePathRewrite, features.SupportHTTPRouteHostRewrite, // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/5868 // Temporarily disabled and tracking through the following issue. @@ -47,7 +46,6 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, features.SupportHTTPRouteResponseHeaderModification, - features.SupportHTTPRoutePathRewrite, features.SupportHTTPRouteHostRewrite, // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/5868 // Temporarily disabled and tracking through the following issue. From d21bca9b0262b4f603f94614c9ad9d4a6f2367da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 22:37:28 +0200 Subject: [PATCH 21/48] chore(deps): bump github.com/kong/go-database-reconciler (#6108) Bumps [github.com/kong/go-database-reconciler](https://github.com/kong/go-database-reconciler) from 1.12.0 to 1.12.1. - [Release notes](https://github.com/kong/go-database-reconciler/releases) - [Commits](https://github.com/kong/go-database-reconciler/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: github.com/kong/go-database-reconciler dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9a4337ad63..198eedc81f 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/jpillora/backoff v1.0.0 - github.com/kong/go-database-reconciler v1.12.0 + github.com/kong/go-database-reconciler v1.12.1 github.com/kong/go-kong v0.55.0 github.com/kong/kubernetes-telemetry v0.1.3 github.com/kong/kubernetes-testing-framework v0.47.0 diff --git a/go.sum b/go.sum index 065f4a85b3..aac0e9ec18 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kong/go-database-reconciler v1.12.0 h1:8+mt2VX/j5uTByPF0dC7Xsh9LscaPjxxXErzaKbL2ik= -github.com/kong/go-database-reconciler v1.12.0/go.mod h1:fX9SV2ukbuUdVw2h1rakWTi/DUxAV9cbj4QWttoiRrc= +github.com/kong/go-database-reconciler v1.12.1 h1:v6CjU2eo9gAnw+CNQg6r+lW66vuU34XddnVZhd91MdU= +github.com/kong/go-database-reconciler v1.12.1/go.mod h1:fX9SV2ukbuUdVw2h1rakWTi/DUxAV9cbj4QWttoiRrc= github.com/kong/go-kong v0.55.0 h1:lonKRzsDGk12dh9E+y+pWnY2ThXhKuMHjzBHSpCvQLw= github.com/kong/go-kong v0.55.0/go.mod h1:i1cMgTu6RYPHSyMpviShddRnc+DML/vlpgKC00hr8kU= github.com/kong/kubernetes-telemetry v0.1.3 h1:Hz2tkHGIIUqbn1x46QRDmmNjbEtJyxyOvHSPne3uPto= From 20005e9f0aa3f3add17b5a921e2f859cca7ccb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Mon, 3 Jun 2024 07:44:17 +0200 Subject: [PATCH 22/48] conformance: enable HTTPRouteHostRewrite test (#6112) --- test/conformance/gateway_conformance_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 1df08ba212..014bf6b3a6 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -32,6 +32,7 @@ var traditionalRoutesSupportedFeatures = []features.SupportedFeature{ features.SupportHTTPRoute, // extended features features.SupportHTTPRouteResponseHeaderModification, + features.SupportHTTPRoutePathRewrite, features.SupportHTTPRouteHostRewrite, // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/5868 // Temporarily disabled and tracking through the following issue. @@ -46,6 +47,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, features.SupportHTTPRouteResponseHeaderModification, + features.SupportHTTPRoutePathRewrite, features.SupportHTTPRouteHostRewrite, // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/5868 // Temporarily disabled and tracking through the following issue. From 95af584ab560822a40ee6a1dd66a2005f9663e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Mon, 3 Jun 2024 14:31:05 +0200 Subject: [PATCH 23/48] feat: add metrics for FallbackConfiguration (#6105) * feat: add metrics for FallbackConfiguration * address review comments --- CHANGELOG.md | 11 + internal/dataplane/kong_client.go | 18 +- internal/dataplane/sendconfig/sendconfig.go | 13 +- internal/metrics/prometheus.go | 275 ++++++++++++++++++-- internal/metrics/prometheus_test.go | 3 +- test/envtest/metrics_envtest_test.go | 159 +++++++---- test/mocks/admin_api_handler.go | 17 +- 7 files changed, 415 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf4f8e1cc..331a632ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -148,6 +148,17 @@ Adding a new version? You'll need three changes: - Added `FallbackKongConfigurationSucceeded`, `FallbackKongConfigurationTranslationFailed` and `FallbackKongConfigurationApplyFailed` Kubernetes Events to report the status of the fallback configuration. [#6099](https://github.com/Kong/kubernetes-ingress-controller/pull/6099) +- Added Prometheus metrics covering `FallbackConfiguration` feature: + - `ingress_controller_fallback_translation_count` + - `ingress_controller_fallback_translation_broken_resource_count` + - `ingress_controller_fallback_configuration_push_count` + - `ingress_controller_fallback_configuration_push_last` + - `ingress_controller_fallback_configuration_push_duration_milliseconds` + - `ingress_controller_fallback_configuration_push_broken_resource_count` + - `ingress_controller_fallback_cache_generating_duration_milliseconds` + - `ingress_controller_processed_config_snapshot_cache_hit` + - `ingress_controller_processed_config_snapshot_cache_miss` + [#6105](https://github.com/Kong/kubernetes-ingress-controller/pull/6105) - Add support for Kubernetes Gateway API v1.1: - add a flag `--enable-controller-gwapi-grpcroute` to control whether enable or disable GRPCRoute controller. - add support for `GRPCRoute` v1, which requires users to upgrade the Gateway API's CRD to v1.1. diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 187a1cbedd..e72b1ddb2b 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -446,9 +446,12 @@ func (c *KongClient) Update(ctx context.Context) error { // Empty snapshot hash means that the cache hasn't changed since the last snapshot was taken. That optimization can be used // in main code path to avoid unnecessary processing. TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6095 if newSnapshotHash == store.SnapshotHashEmpty { + c.prometheusMetrics.RecordProcessedConfigSnapshotCacheHit() c.logger.V(util.DebugLevel).Info("No configuration change; pushing config to gateway is not necessary, skipping") return nil } + + c.prometheusMetrics.RecordProcessedConfigSnapshotCacheMiss() c.lastProcessedSnapshotHash = newSnapshotHash c.kongConfigBuilder.UpdateCache(cacheSnapshot) } @@ -575,12 +578,14 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( if failuresCount := len(fallbackParsingResult.TranslationFailures); failuresCount > 0 { c.recordResourceFailureEvents(fallbackParsingResult.TranslationFailures, FallbackKongConfigurationTranslationFailedEventReason) + c.prometheusMetrics.RecordFallbackTranslationBrokenResources(failuresCount) + c.prometheusMetrics.RecordFallbackTranslationFailure() c.logger.V(util.DebugLevel).Info("Translation failures occurred when building fallback data-plane configuration", "count", failuresCount) + } else { + c.prometheusMetrics.RecordFallbackTranslationBrokenResources(0) + c.prometheusMetrics.RecordFallbackTranslationSuccess() } - // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6082 - // Expose Prometheus metrics for fallback configuration parsing result. - const isFallback = true _, gatewaysSyncErr = c.sendOutToGatewayClients(ctx, fallbackParsingResult.KongState, c.kongConfig, isFallback) if gatewaysSyncErr != nil { @@ -603,7 +608,11 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( func (c *KongClient) generateFallbackCache( currentCache store.CacheStores, brokenObjects []fallback.ObjectHash, -) (store.CacheStores, error) { +) (s store.CacheStores, err error) { + start := time.Now() + defer func() { + c.prometheusMetrics.RecordFallbackCacheGenerationDuration(time.Since(start), err) + }() if c.kongConfig.UseLastValidConfigForFallback { return c.fallbackConfigGenerator.GenerateBackfillingBrokenObjects( currentCache, @@ -773,6 +782,7 @@ func (c *KongClient) sendToClient( c.prometheusMetrics, c.updateStrategyResolver, c.configChangeDetector, + isFallback, ) // Only record events on applying configuration to Kong gateway here. // Nil error is expected to be passed to indicate success. diff --git a/internal/dataplane/sendconfig/sendconfig.go b/internal/dataplane/sendconfig/sendconfig.go index bae6e66439..d6fd1174c4 100644 --- a/internal/dataplane/sendconfig/sendconfig.go +++ b/internal/dataplane/sendconfig/sendconfig.go @@ -44,6 +44,7 @@ func PerformUpdate( promMetrics *metrics.CtrlFuncMetrics, updateStrategyResolver UpdateStrategyResolver, configChangeDetector ConfigurationChangeDetector, + isFallback bool, ) ([]byte, error) { oldSHA := client.LastConfigSHA() newSHA, err := deckgen.GenerateSHA(targetContent) @@ -81,7 +82,11 @@ func PerformUpdate( // For UpdateError, record the failure and return the error. var updateError UpdateError if errors.As(err, &updateError) { - promMetrics.RecordPushFailure(metricsProtocol, duration, client.BaseRootURL(), len(updateError.ResourceFailures()), updateError.err) + if isFallback { + promMetrics.RecordFallbackPushFailure(metricsProtocol, duration, client.BaseRootURL(), len(updateError.ResourceFailures()), updateError.err) + } else { + promMetrics.RecordPushFailure(metricsProtocol, duration, client.BaseRootURL(), len(updateError.ResourceFailures()), updateError.err) + } return nil, updateError } @@ -89,7 +94,11 @@ func PerformUpdate( return nil, fmt.Errorf("config update failed: %w", err) } - promMetrics.RecordPushSuccess(metricsProtocol, duration, client.BaseRootURL()) + if isFallback { + promMetrics.RecordFallbackPushSuccess(metricsProtocol, duration, client.BaseRootURL()) + } else { + promMetrics.RecordPushSuccess(metricsProtocol, duration, client.BaseRootURL()) + } if client.IsKonnect() { logger.V(util.InfoLevel).Info("Successfully synced configuration to Konnect") diff --git a/internal/metrics/prometheus.go b/internal/metrics/prometheus.go index 11194bd93b..f007da1520 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/prometheus.go @@ -16,17 +16,24 @@ import ( // descriptions of these metrics are found below, where their help text is set in NewCtrlFuncMetrics() type CtrlFuncMetrics struct { - ConfigPushCount *prometheus.CounterVec - - ConfigPushBrokenResources *prometheus.GaugeVec - - TranslationCount *prometheus.CounterVec - + // Regular config push metrics. + ConfigPushCount *prometheus.CounterVec + ConfigPushBrokenResources *prometheus.GaugeVec + TranslationCount *prometheus.CounterVec TranslationBrokenResources prometheus.Gauge - - ConfigPushDuration *prometheus.HistogramVec - - ConfigPushSuccessTime *prometheus.GaugeVec + ConfigPushDuration *prometheus.HistogramVec + ConfigPushSuccessTime *prometheus.GaugeVec + + // Fallback config push metrics. + FallbackTranslationCount *prometheus.CounterVec + FallbackTranslationBrokenResources prometheus.Gauge + FallbackConfigPushCount *prometheus.CounterVec + FallbackConfigPushSuccessTime *prometheus.GaugeVec + FallbackConfigPushBrokenResources *prometheus.GaugeVec + FallbackConfigPushDuration *prometheus.HistogramVec + FallbackCacheGeneratingDuration *prometheus.HistogramVec + ProcessedConfigSnapshotCacheHit prometheus.Counter + ProcessedConfigSnapshotCacheMiss prometheus.Counter } const ( @@ -70,6 +77,7 @@ const ( DataplaneKey string = "dataplane" ) +// Regular config push metrics names. const ( MetricNameConfigPushCount = "ingress_controller_configuration_push_count" MetricNameConfigPushBrokenResources = "ingress_controller_configuration_push_broken_resource_count" @@ -79,6 +87,19 @@ const ( MetricNameConfigPushDuration = "ingress_controller_configuration_push_duration_milliseconds" ) +// Fallback config push metrics names. +const ( + MetricNameFallbackTranslationCount = "ingress_controller_fallback_translation_count" + MetricNameFallbackTranslationBrokenResources = "ingress_controller_fallback_translation_broken_resource_count" + MetricNameFallbackConfigPushCount = "ingress_controller_fallback_configuration_push_count" + MetricNameFallbackConfigPushSuccessTime = "ingress_controller_fallback_configuration_push_last" + MetricNameFallbackConfigPushDuration = "ingress_controller_fallback_configuration_push_duration_milliseconds" + MetricNameFallbackConfigPushBrokenResources = "ingress_controller_fallback_configuration_push_broken_resource_count" + MetricNameFallbackCacheGenerationDuration = "ingress_controller_fallback_cache_generation_duration_milliseconds" + MetricNameProcessedConfigSnapshotCacheHit = "ingress_controller_processed_config_snapshot_cache_hit" + MetricNameProcessedConfigSnapshotCacheMiss = "ingress_controller_processed_config_snapshot_cache_miss" +) + var _lock sync.Mutex func NewCtrlFuncMetrics() *CtrlFuncMetrics { @@ -168,21 +189,133 @@ func NewCtrlFuncMetrics() *CtrlFuncMetrics { []string{DataplaneKey}, ) - metrics.Registry.Unregister(controllerMetrics.ConfigPushCount) - metrics.Registry.Unregister(controllerMetrics.ConfigPushBrokenResources) - metrics.Registry.Unregister(controllerMetrics.TranslationCount) - metrics.Registry.Unregister(controllerMetrics.TranslationBrokenResources) - metrics.Registry.Unregister(controllerMetrics.ConfigPushDuration) - metrics.Registry.Unregister(controllerMetrics.ConfigPushSuccessTime) + controllerMetrics.FallbackTranslationCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricNameFallbackTranslationCount, + Help: fmt.Sprintf("Count of translations from Kubernetes state to Kong state in fallback mode. "+ + "`%s` describes whether there were unrecoverable errors (`%s`) or not (`%s`). "+ + "Unrecoverable error in this case means KIC wasn't able to translate a Kubernetes object to Kong model.", + SuccessKey, SuccessFalse, SuccessTrue, + ), + }, + []string{SuccessKey}, + ) + + controllerMetrics.FallbackTranslationBrokenResources = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: MetricNameFallbackTranslationBrokenResources, + Help: fmt.Sprintf("The number of resources that the controller cannot successfully translate to Kong " + + "configuration in fallback mode.", + ), + }, + ) + + controllerMetrics.FallbackConfigPushCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricNameFallbackConfigPushCount, + Help: fmt.Sprintf( + "Count of successful/failed fallback configuration pushes to Kong. "+ + "`%s` describes the dataplane that was the target of configuration push. "+ + "`%s` describes the configuration protocol (`%s` or `%s`) in use. "+ + "`%s` describes whether there were unrecoverable errors (`%s`) or not (`%s`). "+ + "`%s` is populated in case of `%s=\"%s\"` and describes the reason of failure "+ + "(one of `%s`, `%s`, `%s`).", + DataplaneKey, + ProtocolKey, ProtocolDBLess, ProtocolDeck, + SuccessKey, SuccessFalse, SuccessTrue, + FailureReasonKey, SuccessKey, SuccessFalse, + FailureReasonConflict, FailureReasonNetwork, FailureReasonOther, + ), + }, + []string{SuccessKey, ProtocolKey, FailureReasonKey, DataplaneKey}, + ) + + controllerMetrics.FallbackConfigPushSuccessTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: MetricNameFallbackConfigPushSuccessTime, + Help: fmt.Sprintf("The time of the last successful fallback configuration push. "+ + "`%s` describes the dataplane that was the target of the configuration push.", + DataplaneKey, + ), + }, + []string{DataplaneKey}, + ) + + controllerMetrics.FallbackConfigPushDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: MetricNameFallbackConfigPushDuration, + Help: fmt.Sprintf( + "How long it took to push the fallback configuration to Kong, in milliseconds. "+ + "`%s` describes the dataplane that was the target of configuration push. "+ + "`%s` describes the configuration protocol (`%s` or `%s`) in use. "+ + "`%s` describes whether there were unrecoverable errors (`%s`) or not (`%s`).", + DataplaneKey, + ProtocolKey, ProtocolDBLess, ProtocolDeck, + SuccessKey, SuccessFalse, SuccessTrue, + ), + }, + []string{SuccessKey, ProtocolKey, DataplaneKey}, + ) + + controllerMetrics.FallbackConfigPushBrokenResources = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: MetricNameFallbackConfigPushBrokenResources, + Help: fmt.Sprintf("The number of resources not accepted by Kong when attempting to push "+ + "fallback configuration. `%s` describes the dataplane that was the target of the configuration push.", + DataplaneKey, + ), + }, + []string{DataplaneKey}, + ) + + controllerMetrics.FallbackCacheGeneratingDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: MetricNameFallbackCacheGenerationDuration, + Help: fmt.Sprintf("How long it took to generate a fallback cache, in milliseconds. "+ + "`%s` describes whether the cache generation was successful (`%s`) or not (`%s`).", + SuccessKey, SuccessTrue, SuccessFalse, + ), + }, + []string{SuccessKey}, + ) + + controllerMetrics.ProcessedConfigSnapshotCacheHit = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: MetricNameProcessedConfigSnapshotCacheHit, + Help: "The number of times the controller hit the processed config snapshot cache and skipped generating " + + "a new one.", + }, + ) + + controllerMetrics.ProcessedConfigSnapshotCacheMiss = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: MetricNameProcessedConfigSnapshotCacheMiss, + Help: "The number of times the controller missed the processed config snapshot cache and had to generate " + + "a new one.", + }, + ) - metrics.Registry.MustRegister( + allMetrics := []prometheus.Collector{ controllerMetrics.ConfigPushCount, controllerMetrics.ConfigPushBrokenResources, controllerMetrics.TranslationCount, controllerMetrics.TranslationBrokenResources, controllerMetrics.ConfigPushDuration, controllerMetrics.ConfigPushSuccessTime, - ) + controllerMetrics.FallbackTranslationBrokenResources, + controllerMetrics.FallbackTranslationCount, + controllerMetrics.FallbackConfigPushCount, + controllerMetrics.FallbackConfigPushSuccessTime, + controllerMetrics.FallbackConfigPushDuration, + controllerMetrics.FallbackConfigPushBrokenResources, + controllerMetrics.FallbackCacheGeneratingDuration, + controllerMetrics.ProcessedConfigSnapshotCacheHit, + controllerMetrics.ProcessedConfigSnapshotCacheMiss, + } + for _, m := range allMetrics { + metrics.Registry.Unregister(m) + metrics.Registry.MustRegister(m) + } return controllerMetrics } @@ -223,6 +356,63 @@ func (c *CtrlFuncMetrics) RecordTranslationBrokenResources(count int) { c.TranslationBrokenResources.Set(float64(count)) } +// RecordFallbackTranslationFailure records a failed fallback configuration translation. +func (c *CtrlFuncMetrics) RecordFallbackTranslationFailure() { + c.FallbackTranslationCount.With(prometheus.Labels{ + SuccessKey: SuccessFalse, + }).Inc() +} + +// RecordFallbackTranslationSuccess records a failed fallback configuration translation. +func (c *CtrlFuncMetrics) RecordFallbackTranslationSuccess() { + c.FallbackTranslationCount.With(prometheus.Labels{ + SuccessKey: SuccessTrue, + }).Inc() +} + +// RecordProcessedConfigSnapshotCacheHit records a hit on the processed config snapshot cache. +func (c *CtrlFuncMetrics) RecordProcessedConfigSnapshotCacheHit() { + c.ProcessedConfigSnapshotCacheHit.Inc() +} + +// RecordProcessedConfigSnapshotCacheMiss records a miss on the processed config snapshot cache. +func (c *CtrlFuncMetrics) RecordProcessedConfigSnapshotCacheMiss() { + c.ProcessedConfigSnapshotCacheMiss.Inc() +} + +// RecordFallbackTranslationBrokenResources records the number of fallback resources failing translation. +func (c *CtrlFuncMetrics) RecordFallbackTranslationBrokenResources(count int) { + c.FallbackTranslationBrokenResources.Set(float64(count)) +} + +// RecordFallbackPushSuccess records a successful fallback configuration push. +func (c *CtrlFuncMetrics) RecordFallbackPushSuccess(p Protocol, duration time.Duration, dataplane string) { + dpOpt := withDataplane(dataplane) + c.recordFallbackPushCount(p, dpOpt) + c.recordFallbackPushDuration(p, duration, dpOpt) + c.recordFallbackPushSuccessTime(dpOpt) + c.recordFallbackPushBrokenResources(0, dpOpt) +} + +// RecordFallbackPushFailure records a failed fallback configuration push. +func (c *CtrlFuncMetrics) RecordFallbackPushFailure(p Protocol, duration time.Duration, dataplane string, brokenResourcesCount int, err error) { + dpOpt := withDataplane(dataplane) + c.recordFallbackPushDuration(p, duration, dpOpt, withFailure()) + c.recordFallbackPushCount(p, dpOpt, withError(err)) + c.recordFallbackPushBrokenResources(brokenResourcesCount, dpOpt) +} + +// RecordFallbackCacheGenerationDuration records the duration of a fallback cache generation. +func (c *CtrlFuncMetrics) RecordFallbackCacheGenerationDuration(d time.Duration, err error) { + labels := prometheus.Labels{ + SuccessKey: SuccessTrue, + } + if err != nil { + labels[SuccessKey] = SuccessFalse + } + c.FallbackCacheGeneratingDuration.With(labels).Observe(float64(d.Milliseconds())) +} + type recordOption func(prometheus.Labels) prometheus.Labels func withError(err error) recordOption { @@ -296,6 +486,55 @@ func (c *CtrlFuncMetrics) recordPushSuccessTime(opts ...recordOption) { c.ConfigPushSuccessTime.With(labels).SetToCurrentTime() } +func (c *CtrlFuncMetrics) recordFallbackPushCount(p Protocol, opts ...recordOption) { + labels := prometheus.Labels{ + // Although this is hardcoded to true here, the withError or withFailure opt function will flip it to false. + SuccessKey: SuccessTrue, + ProtocolKey: string(p), + FailureReasonKey: "", + } + + for _, opt := range opts { + labels = opt(labels) + } + + c.FallbackConfigPushCount.With(labels).Inc() +} + +func (c *CtrlFuncMetrics) recordFallbackPushSuccessTime(opts ...recordOption) { + labels := prometheus.Labels{} + + for _, opt := range opts { + labels = opt(labels) + } + + c.FallbackConfigPushSuccessTime.With(labels).SetToCurrentTime() +} + +func (c *CtrlFuncMetrics) recordFallbackPushDuration(p Protocol, d time.Duration, opts ...recordOption) { + labels := prometheus.Labels{ + // Although this is hardcoded to true here, the withError or withFailure opt function will flip it to false. + SuccessKey: SuccessTrue, + ProtocolKey: string(p), + } + + for _, opt := range opts { + labels = opt(labels) + } + + c.FallbackConfigPushDuration.With(labels).Observe(float64(d.Milliseconds())) +} + +func (c *CtrlFuncMetrics) recordFallbackPushBrokenResources(brokenObjectsCount int, opts ...recordOption) { + labels := prometheus.Labels{} + + for _, opt := range opts { + labels = opt(labels) + } + + c.FallbackConfigPushBrokenResources.With(labels).Set(float64(brokenObjectsCount)) +} + // pushFailureReason extracts config push failure reason from an error returned // from sendconfig's onUpdateInMemoryMode or onUpdateDBMode. func pushFailureReason(err error) string { diff --git a/internal/metrics/prometheus_test.go b/internal/metrics/prometheus_test.go index e8c4b64e9d..70702ed917 100644 --- a/internal/metrics/prometheus_test.go +++ b/internal/metrics/prometheus_test.go @@ -33,8 +33,7 @@ func TestRecordPush(t *testing.T) { }) t.Run("recording push failure works", func(t *testing.T) { require.NotPanics(t, func() { - m.RecordPushFailure(ProtocolDBLess, time.Millisecond, "https://10.0.0.1:8080", 5, - fmt.Errorf("custom error")) + m.RecordPushFailure(ProtocolDBLess, time.Millisecond, "https://10.0.0.1:8080", 5, fmt.Errorf("custom error")) }) }) } diff --git a/test/envtest/metrics_envtest_test.go b/test/envtest/metrics_envtest_test.go index af0df4f2f5..db0793ddd7 100644 --- a/test/envtest/metrics_envtest_test.go +++ b/test/envtest/metrics_envtest_test.go @@ -13,7 +13,10 @@ import ( "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/require" + "github.com/kong/kubernetes-ingress-controller/v3/internal/manager" + "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/featuregates" "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" + "github.com/kong/kubernetes-ingress-controller/v3/test/mocks" ) func TestMetricsAreServed(t *testing.T) { @@ -28,65 +31,113 @@ func TestMetricsAreServed(t *testing.T) { scheme := Scheme(t, WithKong) envcfg := Setup(t, scheme) - ctx, cancel := context.WithTimeout(context.Background(), waitTime) - defer cancel() - - cfg, _ := RunManager(ctx, t, envcfg, - AdminAPIOptFns(), - ) - - wantMetrics := []string{ - metrics.MetricNameConfigPushCount, - metrics.MetricNameConfigPushBrokenResources, - metrics.MetricNameTranslationCount, - metrics.MetricNameTranslationBrokenResources, - metrics.MetricNameConfigPushDuration, - metrics.MetricNameConfigPushSuccessTime, + testCases := []struct { + name string + withPushError bool + fallbackConfigurationEnabled bool + expectedMetrics []string + }{ + { + name: "with push error and FallbackConfiguration enabled", + withPushError: true, + fallbackConfigurationEnabled: true, + expectedMetrics: []string{ + metrics.MetricNameConfigPushCount, + metrics.MetricNameConfigPushBrokenResources, + metrics.MetricNameTranslationCount, + metrics.MetricNameTranslationBrokenResources, + metrics.MetricNameConfigPushDuration, + + metrics.MetricNameFallbackTranslationBrokenResources, + metrics.MetricNameFallbackTranslationCount, + metrics.MetricNameFallbackConfigPushCount, + metrics.MetricNameFallbackConfigPushSuccessTime, + metrics.MetricNameFallbackConfigPushDuration, + metrics.MetricNameFallbackConfigPushBrokenResources, + metrics.MetricNameProcessedConfigSnapshotCacheHit, + metrics.MetricNameProcessedConfigSnapshotCacheMiss, + }, + }, + { + name: "without push error", + withPushError: false, + expectedMetrics: []string{ + metrics.MetricNameConfigPushCount, + metrics.MetricNameConfigPushBrokenResources, + metrics.MetricNameTranslationCount, + metrics.MetricNameTranslationBrokenResources, + metrics.MetricNameConfigPushDuration, + metrics.MetricNameConfigPushSuccessTime, + }, + }, } - metricsURL := fmt.Sprintf("http://%s/metrics", cfg.MetricsAddr) - t.Logf("waiting for metrics to be available at %q", metricsURL) - - for _, metric := range wantMetrics { - metric := metric - t.Run(metric, func(t *testing.T) { - require.NoError(t, - retry.Do(func() error { - resp, err := http.Get(metricsURL) - if err != nil { - return fmt.Errorf("error %w checking %q", err, metricsURL) - } - - defer resp.Body.Close() - if http.StatusOK != resp.StatusCode { - return fmt.Errorf("status code %v not as expected (200)", resp.StatusCode) - } - - var parser expfmt.TextParser - mf, err := parser.TextToMetricFamilies(resp.Body) - if err != nil { - return fmt.Errorf("error %w parsing %q", err, metricsURL) - } - - if _, ok := mf[metric]; !ok { - return fmt.Errorf("metric %q not present yet", metric) - } - return nil + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), waitTime) + defer cancel() + + var adminAPIOpts []mocks.AdminAPIHandlerOpt + if tc.withPushError { + adminAPIOpts = append(adminAPIOpts, + mocks.WithConfigPostError([]byte(`{"flattened_errors": [{"errors": [{"messages": ["broken object"]}], "entity_tags": ["k8s-name:test-service","k8s-namespace:default","k8s-kind:Service","k8s-uid:a3b8afcc-9f19-42e4-aa8f-5866168c2ad3","k8s-group:","k8s-version:v1"]}]}`)), + mocks.WithConfigPostErrorOnlyOnFirstRequest(), + ) + } + cfg, _ := RunManager(ctx, t, envcfg, + AdminAPIOptFns(adminAPIOpts...), + func(cfg *manager.Config) { + cfg.FeatureGates[featuregates.FallbackConfiguration] = tc.fallbackConfigurationEnabled }, - retry.Context(ctx), - retry.Delay(tickTime), - retry.MaxDelay(maxDelay), - retry.MaxJitter(maxDelay), - retry.DelayType(retry.BackOffDelay), - retry.Attempts(0), // We're using a context with timeout, so we don't need to limit the number of attempts. - retry.LastErrorOnly(true), - retry.OnRetry(func(_ uint, err error) { - t.Logf("metric %s not present yet, err: %v", metric, err.Error()) - }), - ), ) - t.Logf("metric %q is present at /metrics", metric) + wantMetrics := tc.expectedMetrics + + metricsURL := fmt.Sprintf("http://%s/metrics", cfg.MetricsAddr) + t.Logf("waiting for metrics to be available at %q", metricsURL) + + for _, metric := range wantMetrics { + metric := metric + t.Run(metric, func(t *testing.T) { + require.NoError(t, + retry.Do(func() error { + resp, err := http.Get(metricsURL) + if err != nil { + return fmt.Errorf("error %w checking %q", err, metricsURL) + } + + defer resp.Body.Close() + if http.StatusOK != resp.StatusCode { + return fmt.Errorf("status code %v not as expected (200)", resp.StatusCode) + } + + var parser expfmt.TextParser + mf, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { + return fmt.Errorf("error %w parsing %q", err, metricsURL) + } + + if _, ok := mf[metric]; !ok { + return fmt.Errorf("metric %q not present yet", metric) + } + return nil + }, + retry.Context(ctx), + retry.Delay(tickTime), + retry.MaxDelay(maxDelay), + retry.MaxJitter(maxDelay), + retry.DelayType(retry.BackOffDelay), + retry.Attempts(0), // We're using a context with timeout, so we don't need to limit the number of attempts. + retry.LastErrorOnly(true), + retry.OnRetry(func(_ uint, err error) { + t.Logf("metric %s not present yet, err: %v", metric, err.Error()) + }), + ), + ) + + t.Logf("metric %q is present at /metrics", metric) + }) + } }) } } diff --git a/test/mocks/admin_api_handler.go b/test/mocks/admin_api_handler.go index 57905a5129..9ba3affd8e 100644 --- a/test/mocks/admin_api_handler.go +++ b/test/mocks/admin_api_handler.go @@ -53,6 +53,13 @@ type AdminAPIHandler struct { // responding to a `POST /config` request. configPostErrorBody []byte + // configPostErrorOnlyOnFirstRequest is a flag that indicates whether the error should be returned only on the first + // request to `POST /config`. + configPostErrorOnlyOnFirstRequest bool + + // configPostCalled is a flag that indicates whether the `POST /config` endpoint was called. + configPostCalled bool + // rootResponse is the response body served by the admin API root "GET /" endpoint. rootResponse []byte } @@ -100,6 +107,12 @@ func WithRoot(response []byte) AdminAPIHandlerOpt { } } +func WithConfigPostErrorOnlyOnFirstRequest() AdminAPIHandlerOpt { + return func(h *AdminAPIHandler) { + h.configPostErrorOnlyOnFirstRequest = true + } +} + func NewAdminAPIHandler(t *testing.T, opts ...AdminAPIHandlerOpt) *AdminAPIHandler { h := &AdminAPIHandler{ version: versions.KICv3VersionCutoff.String(), @@ -177,7 +190,8 @@ func NewAdminAPIHandler(t *testing.T, opts ...AdminAPIHandlerOpt) *AdminAPIHandl } case http.MethodPost: - if h.configPostErrorBody != nil { + firstRequestErrorAlreadyReturned := h.configPostErrorOnlyOnFirstRequest && h.configPostCalled + if h.configPostErrorBody != nil && !firstRequestErrorAlreadyReturned { w.WriteHeader(http.StatusBadRequest) _, _ = w.Write(h.configPostErrorBody) } else { @@ -186,6 +200,7 @@ func NewAdminAPIHandler(t *testing.T, opts ...AdminAPIHandlerOpt) *AdminAPIHandl h.t.Logf("got config: %v", string(b)) h.config = b } + h.configPostCalled = true default: t.Errorf("unexpected request: %s %s", r.Method, r.URL) } From c24305af598daecd757164741f231e056c892ac3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:04:07 +0200 Subject: [PATCH 24/48] chore(deps): bump github.com/prometheus/common from 0.53.0 to 0.54.0 (#6117) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.53.0 to 0.54.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.53.0...v0.54.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 198eedc81f..2f0b760bfe 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus/client_golang v1.19.1 - github.com/prometheus/common v0.53.0 + github.com/prometheus/common v0.54.0 github.com/samber/lo v1.39.0 github.com/samber/mo v1.11.0 github.com/sethvargo/go-password v0.3.0 diff --git a/go.sum b/go.sum index aac0e9ec18..a84512c53c 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJL github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU= From 5cefa9a44fe51fe09ebad0a82a7aba3b42bec2fa Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Mon, 3 Jun 2024 18:27:55 +0200 Subject: [PATCH 25/48] chore(tests): introduce isolated TestHTTPRouteWithBrokenPluginFallback (#6118) --- ...eway-httproute-broken-plugin-fallback.yaml | 171 ++++++++++++++++++ examples/gateway-httproute.yaml | 62 ++++--- test/integration/examples_test.go | 50 ----- .../isolated/examples_httproute_test.go | 120 ++++++++++++ test/integration/isolated/suite_test.go | 14 +- 5 files changed, 343 insertions(+), 74 deletions(-) create mode 100644 examples/gateway-httproute-broken-plugin-fallback.yaml create mode 100644 test/integration/isolated/examples_httproute_test.go diff --git a/examples/gateway-httproute-broken-plugin-fallback.yaml b/examples/gateway-httproute-broken-plugin-fallback.yaml new file mode 100644 index 0000000000..0c0d5af77b --- /dev/null +++ b/examples/gateway-httproute-broken-plugin-fallback.yaml @@ -0,0 +1,171 @@ +# This configuration file presents fallback configuration (feature gate FallbackConfiguration=true), +# it contains a plugin that is misconfigured and will not work. The whole route /for-auth-users won't +# be configured. Only the route /httproute-testing will be configured. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: echo-1 + labels: + app: echo-1 +spec: + selector: + matchLabels: + app: echo-1 + template: + metadata: + labels: + app: echo-1 + spec: + containers: + - name: echo-1 + image: kong/go-echo:0.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 80 + resources: + limits: + memory: "64Mi" + cpu: "250m" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: echo-1 + name: echo-1 +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 1027 + selector: + app: echo-1 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: echo-2 + labels: + app: echo-2 +spec: + selector: + matchLabels: + app: echo-2 + template: + metadata: + labels: + app: echo-2 + spec: + containers: + - name: echo-2 + image: kong/go-echo:0.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 80 + resources: + limits: + memory: "64Mi" + cpu: "250m" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: echo-2 + name: echo-2 +spec: + ports: + - port: 8080 + protocol: TCP + targetPort: 1027 + selector: + app: echo-2 + type: ClusterIP +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: kong + annotations: + konghq.com/gatewayclass-unmanaged: "true" +spec: + controllerName: konghq.com/kic-gateway-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: kong +spec: + gatewayClassName: kong + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-testing + annotations: + konghq.com/strip-path: "true" +spec: + parentRefs: + - name: kong + rules: + - matches: + - path: + type: PathPrefix + value: /httproute-testing + backendRefs: + - name: echo-1 + kind: Service + port: 80 + weight: 75 + - name: echo-2 + kind: Service + port: 8080 + weight: 25 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-testing2 + annotations: + konghq.com/strip-path: "true" + konghq.com/plugins: key-auth +spec: + parentRefs: + - name: kong + rules: + - matches: + - path: + type: PathPrefix + value: /for-auth-users + backendRefs: + - name: echo-1 + kind: Service + port: 80 + weight: 75 + - name: echo-2 + kind: Service + port: 8080 + weight: 25 +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: key-auth + namespace: default +plugin: key-auth +config: + # Should be key_names, not keys. + keys: ["key"] diff --git a/examples/gateway-httproute.yaml b/examples/gateway-httproute.yaml index 2f774b7f43..324c29794b 100644 --- a/examples/gateway-httproute.yaml +++ b/examples/gateway-httproute.yaml @@ -5,73 +5,91 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: httpbin + name: echo-1 labels: - app: httpbin + app: echo-1 spec: selector: matchLabels: - app: httpbin + app: echo-1 template: metadata: labels: - app: httpbin + app: echo-1 spec: containers: - - name: httpbin - image: kong/httpbin:0.1.0 + - name: echo-1 + image: kong/go-echo:0.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - containerPort: 80 + resources: + limits: + memory: "64Mi" + cpu: "250m" --- apiVersion: v1 kind: Service metadata: labels: - app: httpbin - name: httpbin + app: echo-1 + name: echo-1 spec: ports: - port: 80 protocol: TCP - targetPort: 80 + targetPort: 1027 selector: - app: httpbin + app: echo-1 type: ClusterIP --- apiVersion: apps/v1 kind: Deployment metadata: - name: nginx + name: echo-2 labels: - app: nginx + app: echo-2 spec: selector: matchLabels: - app: nginx + app: echo-2 template: metadata: labels: - app: nginx + app: echo-2 spec: containers: - - name: nginx - image: nginx + - name: echo-2 + image: kong/go-echo:0.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - containerPort: 80 + resources: + limits: + memory: "64Mi" + cpu: "250m" --- apiVersion: v1 kind: Service metadata: labels: - app: nginx - name: nginx + app: echo-2 + name: echo-2 spec: ports: - port: 8080 protocol: TCP - targetPort: 80 + targetPort: 1027 selector: - app: nginx + app: echo-2 type: ClusterIP --- apiVersion: gateway.networking.k8s.io/v1 @@ -109,11 +127,11 @@ spec: type: PathPrefix value: /httproute-testing backendRefs: - - name: httpbin + - name: echo-1 kind: Service port: 80 weight: 75 - - name: nginx + - name: echo-2 kind: Service port: 8080 weight: 25 diff --git a/test/integration/examples_test.go b/test/integration/examples_test.go index 9b76fca112..a96cca069e 100644 --- a/test/integration/examples_test.go +++ b/test/integration/examples_test.go @@ -16,9 +16,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" - "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/test" "github.com/kong/kubernetes-ingress-controller/v3/test/consts" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" @@ -26,54 +24,6 @@ import ( const examplesDIR = "../../examples" -func TestHTTPRouteExample(t *testing.T) { - var ( - httprouteExampleManifests = fmt.Sprintf("%s/gateway-httproute.yaml", examplesDIR) - ctx = context.Background() - ) - - _, cleaner := helpers.Setup(ctx, t, env) - - t.Logf("configuring test and setting up API clients") - gwc, err := gatewayclient.NewForConfig(env.Cluster().Config()) - require.NoError(t, err) - - t.Logf("applying yaml manifest %s", httprouteExampleManifests) - b, err := os.ReadFile(httprouteExampleManifests) - require.NoError(t, err) - require.NoError(t, clusters.ApplyManifestByYAML(ctx, env.Cluster(), string(b))) - cleaner.AddManifest(string(b)) - - t.Logf("verifying that the Gateway receives listen addresses") - var gatewayIP string - require.Eventually(t, func() bool { - obj, err := gwc.GatewayV1().Gateways(corev1.NamespaceDefault).Get(ctx, "kong", metav1.GetOptions{}) - if err != nil { - return false - } - - for _, addr := range obj.Status.Addresses { - if addr.Type != nil && *addr.Type == gatewayapi.IPAddressType { - gatewayIP = addr.Value - return true - } - } - - return false - }, gatewayUpdateWaitTime, waitTick) - - require.NoError(t, err) - t.Logf("verifying that the HTTPRoute becomes routable") - helpers.EventuallyGETPath( - t, nil, gatewayIP, "/httproute-testing", http.StatusOK, "httpbin.org", nil, ingressWait, waitTick, - ) - - t.Logf("verifying that the backendRefs are being loadbalanced") - helpers.EventuallyGETPath( - t, nil, gatewayIP, "/httproute-testing", http.StatusOK, "Welcome to nginx!", nil, ingressWait, waitTick, - ) -} - func TestTCPRouteExample(t *testing.T) { RunWhenKongExpressionRouter(context.Background(), t) t.Log("locking TCP port") diff --git a/test/integration/isolated/examples_httproute_test.go b/test/integration/isolated/examples_httproute_test.go new file mode 100644 index 0000000000..04200d87be --- /dev/null +++ b/test/integration/isolated/examples_httproute_test.go @@ -0,0 +1,120 @@ +//go:build integration_tests + +package isolated + +import ( + "context" + "net/http" + "os" + "testing" + + "github.com/kong/kubernetes-testing-framework/pkg/clusters" + "github.com/stretchr/testify/assert" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" + "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" + "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" +) + +func TestHTTPRouteExample(t *testing.T) { + httprouteExampleManifest := examplesManifestPath("gateway-httproute.yaml") + + f := features. + New("example"). + WithLabel(testlabels.Example, testlabels.ExampleTrue). + WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). + WithLabel(testlabels.Kind, testlabels.KindHTTPRoute). + WithSetup("deploy kong addon into cluster", featureSetup( + withControllerManagerOpts(helpers.ControllerManagerOptAdditionalWatchNamespace("default")), + )). + Assess("deploying to cluster works and HTTP requests are routed properly", + runHTTPRouteExampleTestScenario(httprouteExampleManifest), + ). + Teardown(featureTeardown()) + + tenv.Test(t, f.Feature()) +} + +func TestHTTPRouteWithBrokenPluginFallback(t *testing.T) { + httprouteWithBrokenPluginFallback := examplesManifestPath("gateway-httproute-broken-plugin-fallback.yaml") + + f := features. + New("example"). + WithLabel(testlabels.Example, testlabels.ExampleTrue). + WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). + WithLabel(testlabels.Kind, testlabels.KindHTTPRoute). + WithSetup("deploy kong addon into cluster", featureSetup( + withControllerManagerOpts( + helpers.ControllerManagerOptAdditionalWatchNamespace("default"), + ), + withControllerManagerFeatureGates(map[string]string{"FallbackConfiguration": "true"}), + )). + Assess("deploying to cluster works and HTTP requests are routed properly", + runHTTPRouteExampleTestScenario(httprouteWithBrokenPluginFallback), + ). + Assess("verify that route with misconfigured plugin is not operational", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + proxyURL := GetHTTPURLFromCtx(ctx) + t.Logf("verifying that Kong gateway response in returned instead of desired site") + helpers.EventuallyGETPath( + t, + proxyURL, + proxyURL.Host, + "/for-auth-users", + http.StatusNotFound, + "no Route matched with those values", + nil, + consts.IngressWait, + consts.WaitTick, + ) + return ctx + }). + Teardown(featureTeardown()) + + tenv.Test(t, f.Feature()) +} + +func runHTTPRouteExampleTestScenario(manifestToUse string) func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + return func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t) + cluster := GetClusterFromCtx(ctx) + proxyURL := GetHTTPURLFromCtx(ctx) + + t.Logf("applying yaml manifest %s", manifestToUse) + manifest, err := os.ReadFile(manifestToUse) + assert.NoError(t, err) + assert.NoError(t, clusters.ApplyManifestByYAML(ctx, cluster, string(manifest))) + cleaner.AddManifest(string(manifest)) + + t.Logf("verifying that traffic is routed properly") + + t.Logf("verifying that the HTTPRoute becomes routable") + helpers.EventuallyGETPath( + t, + proxyURL, + proxyURL.Host, + "/httproute-testing", + http.StatusOK, + "echo-1", + nil, + consts.IngressWait, + consts.WaitTick, + ) + + t.Logf("verifying that the backendRefs are being loadbalanced") + helpers.EventuallyGETPath( + t, + proxyURL, + proxyURL.Host, + "/httproute-testing", + http.StatusOK, + "echo-2", + nil, + consts.IngressWait, + consts.WaitTick, + ) + + return ctx + } +} diff --git a/test/integration/isolated/suite_test.go b/test/integration/isolated/suite_test.go index 9a46da747b..03238a6737 100644 --- a/test/integration/isolated/suite_test.go +++ b/test/integration/isolated/suite_test.go @@ -149,8 +149,9 @@ func TestMain(m *testing.M) { } type featureSetupCfg struct { - controllerManagerOpts []helpers.ControllerManagerOpt - kongProxyEnvVars map[string]string + controllerManagerOpts []helpers.ControllerManagerOpt + controllerManagerFeatureGates map[string]string + kongProxyEnvVars map[string]string } type featureSetupOpt func(*featureSetupCfg) @@ -167,6 +168,12 @@ func withKongProxyEnvVars(envVars map[string]string) featureSetupOpt { } } +func withControllerManagerFeatureGates(gates map[string]string) featureSetupOpt { + return func(o *featureSetupCfg) { + o.controllerManagerFeatureGates = gates + } +} + func featureSetup(opts ...featureSetupOpt) func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { var setupCfg featureSetupCfg for _, opt := range opts { @@ -313,6 +320,9 @@ func featureSetup(opts ...featureSetupOpt) func(ctx context.Context, t *testing. t.Logf("configuring feature gates") // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/4849 featureGates := consts.DefaultFeatureGates + for gate, value := range setupCfg.controllerManagerFeatureGates { + featureGates += "," + fmt.Sprintf("%s=%s", gate, value) + } t.Logf("feature gates enabled: %s", featureGates) t.Logf("starting the controller manager") From 6bb9b0e830ad9121502acb56412b696684cd6ea1 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Tue, 4 Jun 2024 01:23:51 -0700 Subject: [PATCH 26/48] feat(diag) expose fallback diagnostic (#6101) Structure config dump diagnostic metadata. Add additional metadata to diagnostic config dumps indicating whether the dump is a result of a fallback config push and, if so, which objects triggered the fallback push. --- CHANGELOG.md | 7 ++ .../dataplane/fallback/cache_to_graph_test.go | 84 +++++++++---------- internal/dataplane/fallback/graph.go | 19 +++-- internal/dataplane/kong_client.go | 54 +++++++++--- internal/dataplane/kong_client_test.go | 66 +++++++++++++-- internal/diagnostics/server.go | 44 ++++++++-- internal/diagnostics/server_test.go | 5 +- internal/diagnostics/types.go | 62 ++++++++++++++ internal/manager/run.go | 3 +- internal/util/configdiag.go | 22 ----- test/envtest/crds_envtest_test.go | 4 +- test/envtest/ingress_test.go | 18 +++- test/envtest/kongupstreampolicy_test.go | 9 +- test/envtest/telemetry_test.go | 4 +- 14 files changed, 290 insertions(+), 111 deletions(-) create mode 100644 internal/diagnostics/types.go delete mode 100644 internal/util/configdiag.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 331a632ef1..d736b5b99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,13 @@ Adding a new version? You'll need three changes: performance benefits, however, so labeling plugin configuration Secrets and enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) +- The `/debug/config/failed` and `/debug/config/successful` diagnostic + endpoints now nest configuration dumps under a `config` key. These endpoints + previously returned the configuration dump at the root. They now return + additional metadata along with the configuration. This change should not + impact normal usage, but if you scrape these endpoints, be aware that their + output format has changed. + [#6101](https://github.com/Kong/kubernetes-ingress-controller/pull/6101) ### Added diff --git a/internal/dataplane/fallback/cache_to_graph_test.go b/internal/dataplane/fallback/cache_to_graph_test.go index 03f01e7ef3..7fe85ff1db 100644 --- a/internal/dataplane/fallback/cache_to_graph_test.go +++ b/internal/dataplane/fallback/cache_to_graph_test.go @@ -101,15 +101,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { }, ), expectedAdjacencyMap: map[string][]string{ - "Ingress:test-namespace/test-ingress": {}, - "IngressClass:test-ingress-class": { - "Ingress:test-namespace/test-ingress", + "networking.k8s.io/Ingress:test-namespace/test-ingress": {}, + "networking.k8s.io/IngressClass:test-ingress-class": { + "networking.k8s.io/Ingress:test-namespace/test-ingress", }, - "Service:test-namespace/test-service": { - "Ingress:test-namespace/test-ingress", + "core/Service:test-namespace/test-service": { + "networking.k8s.io/Ingress:test-namespace/test-ingress", }, - "KongServiceFacade:test-namespace/test-kong-service-facade": { - "Ingress:test-namespace/test-ingress", + "incubator.ingress-controller.konghq.com/KongServiceFacade:test-namespace/test-kong-service-facade": { + "networking.k8s.io/Ingress:test-namespace/test-ingress", }, }, }, @@ -146,15 +146,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { testKongClusterPlugin(t, "cluster-1"), ), expectedAdjacencyMap: map[string][]string{ - "HTTPRoute:test-namespace/test-route": {}, - "Service:test-namespace/1": { - "HTTPRoute:test-namespace/test-route", + "gateway.networking.k8s.io/HTTPRoute:test-namespace/test-route": {}, + "core/Service:test-namespace/1": { + "gateway.networking.k8s.io/HTTPRoute:test-namespace/test-route", }, - "KongPlugin:test-namespace/1": { - "HTTPRoute:test-namespace/test-route", + "configuration.konghq.com/KongPlugin:test-namespace/1": { + "gateway.networking.k8s.io/HTTPRoute:test-namespace/test-route", }, - "KongClusterPlugin:test-namespace/cluster-1": { - "HTTPRoute:test-namespace/test-route", + "configuration.konghq.com/KongClusterPlugin:test-namespace/cluster-1": { + "gateway.networking.k8s.io/HTTPRoute:test-namespace/test-route", }, }, }, @@ -189,15 +189,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { testKongClusterPlugin(t, "cluster-1"), ), expectedAdjacencyMap: map[string][]string{ - "TLSRoute:test-namespace/test-route": {}, - "Service:test-namespace/1": { - "TLSRoute:test-namespace/test-route", + "gateway.networking.k8s.io/TLSRoute:test-namespace/test-route": {}, + "core/Service:test-namespace/1": { + "gateway.networking.k8s.io/TLSRoute:test-namespace/test-route", }, - "KongPlugin:test-namespace/1": { - "TLSRoute:test-namespace/test-route", + "configuration.konghq.com/KongPlugin:test-namespace/1": { + "gateway.networking.k8s.io/TLSRoute:test-namespace/test-route", }, - "KongClusterPlugin:test-namespace/cluster-1": { - "TLSRoute:test-namespace/test-route", + "configuration.konghq.com/KongClusterPlugin:test-namespace/cluster-1": { + "gateway.networking.k8s.io/TLSRoute:test-namespace/test-route", }, }, }, @@ -232,15 +232,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { testKongClusterPlugin(t, "cluster-1"), ), expectedAdjacencyMap: map[string][]string{ - "TCPRoute:test-namespace/test-route": {}, - "Service:test-namespace/1": { - "TCPRoute:test-namespace/test-route", + "gateway.networking.k8s.io/TCPRoute:test-namespace/test-route": {}, + "core/Service:test-namespace/1": { + "gateway.networking.k8s.io/TCPRoute:test-namespace/test-route", }, - "KongPlugin:test-namespace/1": { - "TCPRoute:test-namespace/test-route", + "configuration.konghq.com/KongPlugin:test-namespace/1": { + "gateway.networking.k8s.io/TCPRoute:test-namespace/test-route", }, - "KongClusterPlugin:test-namespace/cluster-1": { - "TCPRoute:test-namespace/test-route", + "configuration.konghq.com/KongClusterPlugin:test-namespace/cluster-1": { + "gateway.networking.k8s.io/TCPRoute:test-namespace/test-route", }, }, }, @@ -275,15 +275,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { testKongClusterPlugin(t, "cluster-1"), ), expectedAdjacencyMap: map[string][]string{ - "UDPRoute:test-namespace/test-route": {}, - "Service:test-namespace/1": { - "UDPRoute:test-namespace/test-route", + "gateway.networking.k8s.io/UDPRoute:test-namespace/test-route": {}, + "core/Service:test-namespace/1": { + "gateway.networking.k8s.io/UDPRoute:test-namespace/test-route", }, - "KongPlugin:test-namespace/1": { - "UDPRoute:test-namespace/test-route", + "configuration.konghq.com/KongPlugin:test-namespace/1": { + "gateway.networking.k8s.io/UDPRoute:test-namespace/test-route", }, - "KongClusterPlugin:test-namespace/cluster-1": { - "UDPRoute:test-namespace/test-route", + "configuration.konghq.com/KongClusterPlugin:test-namespace/cluster-1": { + "gateway.networking.k8s.io/UDPRoute:test-namespace/test-route", }, }, }, @@ -320,15 +320,15 @@ func TestDefaultCacheGraphProvider_CacheToGraph(t *testing.T) { testKongClusterPlugin(t, "cluster-1"), ), expectedAdjacencyMap: map[string][]string{ - "GRPCRoute:test-namespace/test-route": {}, - "Service:test-namespace/1": { - "GRPCRoute:test-namespace/test-route", + "gateway.networking.k8s.io/GRPCRoute:test-namespace/test-route": {}, + "core/Service:test-namespace/1": { + "gateway.networking.k8s.io/GRPCRoute:test-namespace/test-route", }, - "KongPlugin:test-namespace/1": { - "GRPCRoute:test-namespace/test-route", + "configuration.konghq.com/KongPlugin:test-namespace/1": { + "gateway.networking.k8s.io/GRPCRoute:test-namespace/test-route", }, - "KongClusterPlugin:test-namespace/cluster-1": { - "GRPCRoute:test-namespace/test-route", + "configuration.konghq.com/KongClusterPlugin:test-namespace/cluster-1": { + "gateway.networking.k8s.io/GRPCRoute:test-namespace/test-route", }, }, }, diff --git a/internal/dataplane/fallback/graph.go b/internal/dataplane/fallback/graph.go index 2986995b35..58cbc4364f 100644 --- a/internal/dataplane/fallback/graph.go +++ b/internal/dataplane/fallback/graph.go @@ -34,19 +34,27 @@ type ObjectHash struct { // UID is the unique identifier of the object. UID k8stypes.UID - // Kind, Namespace and Name are the object's kind, namespace and name - included for debugging purposes. - Kind string + // Group is the object's group. + Group string + // Kind is the object's Kind. + Kind string + // Namespace is the object's Namespace. Namespace string - Name string + // Name is the object's Name. + Name string } // String returns a string representation of the ObjectHash. It intentionally does not include the UID // as it is not human-readable and is not necessary for debugging purposes. func (h ObjectHash) String() string { + group := h.Group + if group == "" { + group = "core" + } if h.Namespace == "" { - return fmt.Sprintf("%s:%s", h.Kind, h.Name) + return fmt.Sprintf("%s/%s:%s", group, h.Kind, h.Name) } - return fmt.Sprintf("%s:%s/%s", h.Kind, h.Namespace, h.Name) + return fmt.Sprintf("%s/%s:%s/%s", group, h.Kind, h.Namespace, h.Name) } // GetObjectHash is a function that returns a unique identifier for a given object that is used as a @@ -54,6 +62,7 @@ func (h ObjectHash) String() string { func GetObjectHash(obj client.Object) ObjectHash { return ObjectHash{ UID: obj.GetUID(), + Group: obj.GetObjectKind().GroupVersionKind().Group, Kind: obj.GetObjectKind().GroupVersionKind().Kind, Namespace: obj.GetNamespace(), Name: obj.GetName(), diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index e72b1ddb2b..3ee98b4484 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -34,6 +34,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/kongstate" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/sendconfig" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/translator" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" @@ -102,7 +103,7 @@ type KongClient struct { // diagnostic is the client and configuration for reporting diagnostic // information during data-plane update runtime. - diagnostic util.ConfigDumpDiagnostic + diagnostic diagnostics.ConfigDumpDiagnostic // prometheusMetrics is the client for shipping metrics information // updates to the prometheus exporter. @@ -175,6 +176,9 @@ type KongClient struct { // While lastProcessedSnapshotHash keeps track of the last processed cache snapshot (the one kept in KongClient.cache), // lastValidCacheSnapshot can also represent the fallback cache snapshot that was successfully synced with gateways. lastValidCacheSnapshot store.CacheStores + + // brokenObjects is a list of the Kubernetes resources that failed to sync and triggered a fallback sync. + brokenObjects []fallback.ObjectHash } // NewKongClient provides a new KongClient object after connecting to the @@ -182,7 +186,7 @@ type KongClient struct { func NewKongClient( logger logr.Logger, timeout time.Duration, - diagnostic util.ConfigDumpDiagnostic, + diagnostic diagnostics.ConfigDumpDiagnostic, kongConfig sendconfig.Config, eventRecorder record.EventRecorder, dbMode dpconf.DBMode, @@ -553,6 +557,10 @@ func (c *KongClient) tryRecoveringFromGatewaysSyncError( return nil } +func (c *KongClient) cacheBrokenObjectList(list []fallback.ObjectHash) { + c.brokenObjects = list +} + // tryRecoveringWithFallbackConfiguration tries to recover from a configuration rejection by generating a fallback // configuration excluding affected objects from the cache. func (c *KongClient) tryRecoveringWithFallbackConfiguration( @@ -587,6 +595,7 @@ func (c *KongClient) tryRecoveringWithFallbackConfiguration( } const isFallback = true + c.cacheBrokenObjectList(brokenObjects) _, gatewaysSyncErr = c.sendOutToGatewayClients(ctx, fallbackParsingResult.KongState, c.kongConfig, isFallback) if gatewaysSyncErr != nil { return fmt.Errorf("failed to sync fallback configuration with gateways: %w", gatewaysSyncErr) @@ -768,7 +777,7 @@ func (c *KongClient) sendToClient( AppendStubEntityWhenConfigEmpty: !client.IsKonnect() && config.InMemory, } targetContent := deckgen.ToDeckContent(ctx, logger, s, deckGenParams) - sendDiagnostic := prepareSendDiagnosticFn(ctx, logger, c.diagnostic, s, targetContent, deckGenParams) + sendDiagnostic := prepareSendDiagnosticFn(ctx, logger, c.diagnostic, s, targetContent, deckGenParams, c.brokenObjects) // apply the configuration update in Kong timedCtx, cancel := context.WithTimeout(ctx, c.requestTimeout) @@ -805,14 +814,14 @@ func (c *KongClient) sendToClient( if errors.As(err, &responseParsingErr) { rawResponseBody = responseParsingErr.ResponseBody() } - sendDiagnostic(true, rawResponseBody) + sendDiagnostic(diagnostics.DumpMeta{Failed: true, Hash: string(newConfigSHA)}, rawResponseBody) if err := ctx.Err(); err != nil { logger.Error(err, "Exceeded Kong API timeout, consider increasing --proxy-timeout-seconds") } return "", fmt.Errorf("performing update for %s failed: %w", client.BaseRootURL(), err) } - sendDiagnostic(false, nil) // No error occurred. + sendDiagnostic(diagnostics.DumpMeta{Failed: false, Hash: string(newConfigSHA)}, nil) // No error occurred. // update the lastConfigSHA with the new updated checksum client.SetLastConfigSHA(newConfigSHA) @@ -832,21 +841,22 @@ func (c *KongClient) SetConfigStatusNotifier(n clients.ConfigStatusNotifier) { // Dataplane Client - Kong - Private // ----------------------------------------------------------------------------- -type sendDiagnosticFn func(failed bool, raw []byte) +type sendDiagnosticFn func(meta diagnostics.DumpMeta, raw []byte) // prepareSendDiagnosticFn generates sendDiagnosticFn. // Diagnostics are sent only when provided diagnostic config (--dump-config) is set. func prepareSendDiagnosticFn( ctx context.Context, logger logr.Logger, - diagnosticConfig util.ConfigDumpDiagnostic, + diagnosticConfig diagnostics.ConfigDumpDiagnostic, targetState *kongstate.KongState, targetContent *file.Content, deckGenParams deckgen.GenerateDeckContentParams, + broken []fallback.ObjectHash, ) sendDiagnosticFn { - if diagnosticConfig == (util.ConfigDumpDiagnostic{}) { + if diagnosticConfig == (diagnostics.ConfigDumpDiagnostic{}) { // noop, diagnostics won't be sent - return func(bool, []byte) {} + return func(diagnostics.DumpMeta, []byte) {} } var config *file.Content @@ -861,7 +871,7 @@ func prepareSendDiagnosticFn( config = redactedConfig } - return func(failed bool, rawResponseBody []byte) { + return func(meta diagnostics.DumpMeta, rawResponseBody []byte) { // Given that we can send multiple configs to this channel and // the fact that the API that exposes that can only expose 1 config // at a time it means that users utilizing the diagnostics API @@ -869,7 +879,15 @@ func prepareSendDiagnosticFn( // or successfully send configs might be covered by those send // later on but we're OK with this limitation of said API. select { - case diagnosticConfig.Configs <- util.ConfigDump{Failed: failed, Config: *config, RawResponseBody: rawResponseBody}: + case diagnosticConfig.Configs <- diagnostics.ConfigDump{ + Meta: diagnostics.DumpMeta{ + Failed: meta.Failed, + Fallback: len(broken) != 0, + AffectedObjects: hashToAffected(broken), + }, + Config: *config, + RawResponseBody: rawResponseBody, + }: logger.V(util.DebugLevel).Info("Shipping config to diagnostic server") default: logger.Error(nil, "Config diagnostic buffer full, dropping diagnostic config") @@ -877,6 +895,20 @@ func prepareSendDiagnosticFn( } } +func hashToAffected(objs []fallback.ObjectHash) []diagnostics.AffectedObject { + affected := make([]diagnostics.AffectedObject, len(objs)) + for i, obj := range objs { + affected[i] = diagnostics.AffectedObject{ + UID: obj.UID, + Group: obj.Group, + Kind: obj.Kind, + Namespace: obj.Namespace, + Name: obj.Name, + } + } + return affected +} + // triggerKubernetesObjectReport will update the KongClient with a set which // enables filtering for which objects are currently applied to the data-plane, // as well as updating the c.kubernetesObjectStatusQueue to queue those objects diff --git a/internal/dataplane/kong_client_test.go b/internal/dataplane/kong_client_test.go index 931b0eb008..84384f927e 100644 --- a/internal/dataplane/kong_client_test.go +++ b/internal/dataplane/kong_client_test.go @@ -39,9 +39,9 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/kongstate" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/sendconfig" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/translator" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" - "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/internal/versions" kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" "github.com/kong/kubernetes-ingress-controller/v3/test/helpers" @@ -916,7 +916,7 @@ func setupTestKongClient( ) *KongClient { logger := zapr.NewLogger(zap.NewNop()) timeout := time.Second - diagnostic := util.ConfigDumpDiagnostic{} + diagnostic := diagnostics.ConfigDumpDiagnostic{} config := sendconfig.Config{ SanitizeKonnectConfigDumps: true, } @@ -1171,6 +1171,7 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { } configChangeDetector := mockConfigurationChangeDetector{hasConfigurationChanged: true} lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} + diagnosticsCh := make(chan diagnostics.ConfigDump, 10) // make it buffered to avoid blocking // We'll use KongConsumer as an example of a broken object, but it could be any supported type // for the purpose of this test as the fallback config generator is mocked anyway. @@ -1210,13 +1211,15 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + updateStrategyResolver := newMockUpdateStrategyResolver(t) configBuilder := newMockKongConfigBuilder() fallbackConfigGenerator := newMockFallbackConfigGenerator() - updateStrategyResolver := newMockUpdateStrategyResolver(t) kongClient, err := NewKongClient( zapr.NewLogger(zap.NewNop()), time.Second, - util.ConfigDumpDiagnostic{}, + diagnostics.ConfigDumpDiagnostic{ + Configs: diagnosticsCh, + }, sendconfig.Config{ FallbackConfiguration: true, UseLastValidConfigForFallback: tc.enableLastValidConfigFallback, @@ -1310,6 +1313,28 @@ func TestKongClient_FallbackConfiguration_SuccessfulRecovery(t *testing.T) { require.Equal(t, validConsumer.Username, *lastValidConfig.Consumers[0].Username) }) } + + t.Log("Verifying that the last valid config is updated with the config excluding the broken consumer") + lastValidConfig, _ := lastValidConfigFetcher.LastValidConfig() + require.Len(t, lastValidConfig.Consumers, 1) + require.Equal(t, validConsumer.Username, *lastValidConfig.Consumers[0].Username) + + t.Log("Verifying that the diagnostic server received a dump indicating that the broken consumer caused a problem") + // the test will have pushed several successful configs that we don't care about into the diag buffer. this is a + // silly hack to churn through those until we get to the successful fallback + var dump diagnostics.ConfigDump + require.Eventually(t, func() bool { + dump = <-diagnosticsCh + return len(dump.Meta.AffectedObjects) > 0 + }, time.Second, time.Nanosecond) + + // once we have the fallback diagnostic dump, check to confirm that it was a successful fallback push triggered by + // the expected broken consumer + require.False(t, dump.Meta.Failed) + require.True(t, dump.Meta.Fallback) + + require.Equal(t, dump.Meta.AffectedObjects[0].Namespace, brokenConsumer.ObjectMeta.Namespace) + require.Equal(t, dump.Meta.AffectedObjects[0].Name, brokenConsumer.ObjectMeta.Name) } func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing.T) { @@ -1325,6 +1350,7 @@ func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing configBuilder := newMockKongConfigBuilder() lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} fallbackConfigGenerator := newMockFallbackConfigGenerator() + diagnosticsCh := make(chan diagnostics.ConfigDump, 10) // make it buffered to avoid blocking // We'll use KongConsumer as an example of an object, but it could be any supported type // for the purpose of this test as the fallback config generator is mocked anyway. @@ -1345,7 +1371,9 @@ func TestKongClient_FallbackConfiguration_SkipMakingRedundantSnapshot(t *testing kongClient, err := NewKongClient( zapr.NewLogger(zap.NewNop()), time.Second, - util.ConfigDumpDiagnostic{}, + diagnostics.ConfigDumpDiagnostic{ + Configs: diagnosticsCh, + }, sendconfig.Config{ FallbackConfiguration: true, }, @@ -1387,6 +1415,7 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { configBuilder := newMockKongConfigBuilder() lastValidConfigFetcher := &mockKongLastValidConfigFetcher{} fallbackConfigGenerator := newMockFallbackConfigGenerator() + diagnosticsCh := make(chan diagnostics.ConfigDump, 10) // make it buffered to avoid blocking // We'll use KongConsumer as an example of a broken object, but it could be any supported type // for the purpose of this test as the fallback config generator is mocked anyway. @@ -1407,7 +1436,9 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { kongClient, err := NewKongClient( zapr.NewLogger(zap.NewNop()), time.Second, - util.ConfigDumpDiagnostic{}, + diagnostics.ConfigDumpDiagnostic{ + Configs: diagnosticsCh, + }, sendconfig.Config{ FallbackConfiguration: true, }, @@ -1450,6 +1481,23 @@ func TestKongClient_FallbackConfiguration_FailedRecovery(t *testing.T) { t.Log("Verifying that the last valid config is empty") _, hasLastValidConfig := lastValidConfigFetcher.LastValidConfig() require.False(t, hasLastValidConfig, "expected no last valid config to be stored as no successful recovery happened") + + t.Log("Verifying that the diagnostic server received a dump indicating that the broken consumer caused a problem") + // the test will have pushed several successful configs that we don't care about into the diag buffer. this is a + // silly hack to churn through those until we get to the failed fallback + var dump diagnostics.ConfigDump + require.Eventually(t, func() bool { + dump = <-diagnosticsCh + return len(dump.Meta.AffectedObjects) > 0 + }, time.Second, time.Nanosecond) + + // once we have the fallback diagnostic dump, check to confirm that it was a successful fallback push triggered by + // the expected broken consumer + require.True(t, dump.Meta.Failed) + require.True(t, dump.Meta.Fallback) + + require.Equal(t, dump.Meta.AffectedObjects[0].Namespace, brokenConsumer.ObjectMeta.Namespace) + require.Equal(t, dump.Meta.AffectedObjects[0].Name, brokenConsumer.ObjectMeta.Name) } func TestKongClient_LastValidCacheSnapshot(t *testing.T) { @@ -1507,7 +1555,7 @@ func TestKongClient_LastValidCacheSnapshot(t *testing.T) { kongClient, err := NewKongClient( zapr.NewLogger(zap.NewNop()), time.Second, - util.ConfigDumpDiagnostic{}, + diagnostics.ConfigDumpDiagnostic{}, sendconfig.Config{ FallbackConfiguration: tc.fallbackConfigurationFeatureEnabled, UseLastValidConfigForFallback: tc.useLastValidConfigForFallbackEnabled, @@ -1594,8 +1642,8 @@ func TestKongClient_ConfigDumpSanitization(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - diagnosticsCh := make(chan util.ConfigDump, 1) // make it buffered to avoid blocking - kongClient.diagnostic = util.ConfigDumpDiagnostic{ + diagnosticsCh := make(chan diagnostics.ConfigDump, 1) // make it buffered to avoid blocking + kongClient.diagnostic = diagnostics.ConfigDumpDiagnostic{ Configs: diagnosticsCh, DumpsIncludeSensitive: tc.dumpsIncludeSensitive, } diff --git a/internal/diagnostics/server.go b/internal/diagnostics/server.go index babb7a36af..3eaeef9e4a 100644 --- a/internal/diagnostics/server.go +++ b/internal/diagnostics/server.go @@ -31,10 +31,13 @@ const ( type Server struct { logger logr.Logger profilingEnabled bool - configDumps util.ConfigDumpDiagnostic + configDumps ConfigDumpDiagnostic successfulConfigDump file.Content failedConfigDump file.Content + problemObjects []AffectedObject + failedHash string + successHash string rawErrBody []byte configLock *sync.RWMutex } @@ -60,9 +63,9 @@ func NewServer(logger logr.Logger, cfg ServerConfig) Server { } if cfg.ConfigDumpsEnabled { - s.configDumps = util.ConfigDumpDiagnostic{ + s.configDumps = ConfigDumpDiagnostic{ DumpsIncludeSensitive: cfg.DumpSensitiveConfig, - Configs: make(chan util.ConfigDump, diagnosticConfigBufferDepth), + Configs: make(chan ConfigDump, diagnosticConfigBufferDepth), } } @@ -71,14 +74,14 @@ func NewServer(logger logr.Logger, cfg ServerConfig) Server { // ConfigDumps returns an object allowing dumping succeeded and failed configuration updates. // It will return a zero value of the type in case the config dumps are not enabled. -func (s *Server) ConfigDumps() util.ConfigDumpDiagnostic { +func (s *Server) ConfigDumps() ConfigDumpDiagnostic { return s.configDumps } // Listen starts up the HTTP server and blocks until ctx expires. func (s *Server) Listen(ctx context.Context, port int) error { mux := http.NewServeMux() - if s.configDumps != (util.ConfigDumpDiagnostic{}) { + if s.configDumps != (ConfigDumpDiagnostic{}) { s.installDumpHandlers(mux) } if s.profilingEnabled { @@ -121,11 +124,14 @@ func (s *Server) receiveConfig(ctx context.Context) { select { case dump := <-s.configDumps.Configs: s.configLock.Lock() - if dump.Failed { + if dump.Meta.Failed { s.failedConfigDump = dump.Config s.rawErrBody = dump.RawResponseBody + s.problemObjects = dump.Meta.AffectedObjects + s.failedHash = dump.Meta.Hash } else { s.successfulConfigDump = dump.Config + s.successHash = dump.Meta.Hash } s.configLock.Unlock() case <-ctx.Done(): @@ -158,6 +164,7 @@ func installProfilingHandlers(mux *http.ServeMux) { func (s *Server) installDumpHandlers(mux *http.ServeMux) { mux.HandleFunc("/debug/config/successful", s.handleLastValidConfig) mux.HandleFunc("/debug/config/failed", s.handleLastFailedConfig) + mux.HandleFunc("/debug/config/problems", s.handleLastFailedProblemObjects) mux.HandleFunc("/debug/config/raw-error", s.handleLastErrBody) } @@ -172,7 +179,11 @@ func (s *Server) handleLastValidConfig(rw http.ResponseWriter, _ *http.Request) rw.Header().Set("Content-Type", "application/json") s.configLock.RLock() defer s.configLock.RUnlock() - if err := json.NewEncoder(rw).Encode(s.successfulConfigDump); err != nil { + if err := json.NewEncoder(rw).Encode( + configDumpResponse{ + Config: s.successfulConfigDump, + ConfigHash: s.successHash, + }); err != nil { rw.WriteHeader(http.StatusInternalServerError) } } @@ -181,11 +192,28 @@ func (s *Server) handleLastFailedConfig(rw http.ResponseWriter, _ *http.Request) rw.Header().Set("Content-Type", "application/json") s.configLock.RLock() defer s.configLock.RUnlock() - if err := json.NewEncoder(rw).Encode(s.failedConfigDump); err != nil { + if err := json.NewEncoder(rw).Encode( + configDumpResponse{ + Config: s.failedConfigDump, + ConfigHash: s.failedHash, + }); err != nil { rw.WriteHeader(http.StatusInternalServerError) } } +func (s *Server) handleLastFailedProblemObjects(rw http.ResponseWriter, _ *http.Request) { + rw.Header().Set("Content-Type", "application/json") + s.configLock.RLock() + defer s.configLock.RUnlock() + if err := json.NewEncoder(rw).Encode( + problemObjectsResponse{ + ConfigHash: s.failedHash, + BrokenObjects: s.problemObjects, + }); err != nil { + rw.WriteHeader(http.StatusOK) + } +} + func (s *Server) handleLastErrBody(rw http.ResponseWriter, _ *http.Request) { rw.Header().Set("Content-Type", "text/plain") s.configLock.RLock() diff --git a/internal/diagnostics/server_test.go b/internal/diagnostics/server_test.go index f0fa72120e..4e6639e641 100644 --- a/internal/diagnostics/server_test.go +++ b/internal/diagnostics/server_test.go @@ -11,7 +11,6 @@ import ( "github.com/kong/go-database-reconciler/pkg/file" "github.com/stretchr/testify/require" - "github.com/kong/kubernetes-ingress-controller/v3/internal/util" testhelpers "github.com/kong/kubernetes-ingress-controller/v3/test/helpers" ) @@ -47,9 +46,9 @@ func TestDiagnosticsServer_ConfigDumps(t *testing.T) { failed := false for i := 0; i < configDumpsToWrite; i++ { failed = !failed // Toggle failed flag. - configsCh <- util.ConfigDump{ + configsCh <- ConfigDump{ Config: file.Content{}, - Failed: failed, + Meta: DumpMeta{Failed: failed}, RawResponseBody: []byte("fake error body"), } } diff --git a/internal/diagnostics/types.go b/internal/diagnostics/types.go new file mode 100644 index 0000000000..0f526c9235 --- /dev/null +++ b/internal/diagnostics/types.go @@ -0,0 +1,62 @@ +package diagnostics + +import ( + "github.com/kong/go-database-reconciler/pkg/file" + k8stypes "k8s.io/apimachinery/pkg/types" +) + +// DumpMeta annotates a config dump. +type DumpMeta struct { + // Failed indicates the dump was not accepted by the Kong admin API. + Failed bool + // Fallback indicates that the dump is a fallback configuration attempted after a failed config update. + Fallback bool + // AffectedObjects are objects excluded from the fallback configuration. + AffectedObjects []AffectedObject + // Hash is the configuration hash. + Hash string +} + +// ConfigDump contains a config dump and a flag indicating that the config was not successfully applid. +type ConfigDump struct { + // Config is the configuration KIC applied or attempted to apply. + Config file.Content + // Meta contains information about the status and context of the configuration dump. + Meta DumpMeta + // RawResponseBody is the raw Kong Admin API response body from a config apply. It is only available in DB-less mode. + RawResponseBody []byte +} + +type configDumpResponse struct { + ConfigHash string `json:"hash"` + Config file.Content `json:"config"` +} + +type problemObjectsResponse struct { + ConfigHash string `json:"hash"` + BrokenObjects []AffectedObject `json:"brokenObjects"` +} + +// ConfigDumpDiagnostic contains settings and channels for receiving diagnostic configuration dumps. +type ConfigDumpDiagnostic struct { + // DumpsIncludeSensitive is true if the configuration dump includes sensitive values, such as certificate private + // keys and credential secrets. + DumpsIncludeSensitive bool + // Configs is the channel that receives configuration blobs from the configuration update strategy implementation. + Configs chan ConfigDump +} + +// AffectedObject is a Kubernetes object associated with diagnostic information. +type AffectedObject struct { + // UID is the unique identifier of the object. + UID k8stypes.UID + + // Group is the object's group. + Group string + // Kind is the object's Kind. + Kind string + // Namespace is the object's Namespace. + Namespace string + // Name is the object's Name. + Name string +} diff --git a/internal/manager/run.go b/internal/manager/run.go index a56b4ba129..335841972d 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -29,6 +29,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/sendconfig" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/translator" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/konnect" "github.com/kong/kubernetes-ingress-controller/v3/internal/konnect/nodes" @@ -49,7 +50,7 @@ import ( func Run( ctx context.Context, c *Config, - diagnostic util.ConfigDumpDiagnostic, + diagnostic diagnostics.ConfigDumpDiagnostic, logger logr.Logger, ) error { setupLog := ctrl.LoggerFrom(ctx).WithName("setup") diff --git a/internal/util/configdiag.go b/internal/util/configdiag.go deleted file mode 100644 index e615fd0685..0000000000 --- a/internal/util/configdiag.go +++ /dev/null @@ -1,22 +0,0 @@ -package util - -import "github.com/kong/go-database-reconciler/pkg/file" - -// ConfigDump contains a config dump and a flag indicating that the config was not successfully applid. -type ConfigDump struct { - // Config is the configuration KIC applied or attempted to apply. - Config file.Content - // Failed is true if the configuration apply failed. - Failed bool - // RawResponseBody is the raw Kong Admin API response body from a config apply. It is only available in DB-less mode. - RawResponseBody []byte -} - -// ConfigDumpDiagnostic contains settings and channels for receiving diagnostic configuration dumps. -type ConfigDumpDiagnostic struct { - // DumpsIncludeSensitive is true if the configuration dump includes sensitive values, such as certificate private - // keys and credential secrets. - DumpsIncludeSensitive bool - // Configs is the channel that receives configuration blobs from the configuration update strategy implementation. - Configs chan ConfigDump -} diff --git a/test/envtest/crds_envtest_test.go b/test/envtest/crds_envtest_test.go index a1b4f44dd3..70ef403e68 100644 --- a/test/envtest/crds_envtest_test.go +++ b/test/envtest/crds_envtest_test.go @@ -23,8 +23,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v3/internal/annotations" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager" - "github.com/kong/kubernetes-ingress-controller/v3/internal/util" kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1" @@ -105,7 +105,7 @@ func TestNoKongCRDsInstalledIsFatal(t *testing.T) { // Reducing the cache sync timeout to speed up the test. cfg.CacheSyncTimeout = time.Millisecond * 500 - err := manager.Run(ctx, &cfg, util.ConfigDumpDiagnostic{}, logger) + err := manager.Run(ctx, &cfg, diagnostics.ConfigDumpDiagnostic{}, logger) require.ErrorContains(t, err, "timed out waiting for cache to be synced") } diff --git a/test/envtest/ingress_test.go b/test/envtest/ingress_test.go index a81412c4ec..efb0a3b2a8 100644 --- a/test/envtest/ingress_test.go +++ b/test/envtest/ingress_test.go @@ -27,6 +27,15 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/test/helpers" ) +// configDumpResponse mirrors the diagnostics.configDumpResponse struct, which isn't published. +// It's replicated here since some envtests use the config dump endpoints as a hack to extract the config for +// inspection. + +type configDumpResponse struct { + ConfigHash string `json:"hash"` + Config file.Content `json:"config"` +} + func TestIngressWorksWithServiceBackendsSpecifyingOnlyPortNames(t *testing.T) { t.Parallel() @@ -156,15 +165,18 @@ func TestIngressWorksWithServiceBackendsSpecifyingOnlyPortNames(t *testing.T) { defer resp.Body.Close() var ( - config file.Content - buff bytes.Buffer + configDump configDumpResponse + config file.Content + buff bytes.Buffer ) - if err := gojson.NewDecoder(io.TeeReader(resp.Body, &buff)).Decode(&config); err != nil { + if err := gojson.NewDecoder(io.TeeReader(resp.Body, &buff)).Decode(&configDump); err != nil { t.Logf("WARNING: error while decoding config: %+v, response: %s", err, buff.String()) return false } + config = configDump.Config + if len(config.Services) != 1 { t.Logf("WARNING: expected 1 service in config: %+v", config) return false diff --git a/test/envtest/kongupstreampolicy_test.go b/test/envtest/kongupstreampolicy_test.go index a3fd276777..878f767c84 100644 --- a/test/envtest/kongupstreampolicy_test.go +++ b/test/envtest/kongupstreampolicy_test.go @@ -180,15 +180,18 @@ func TestKongUpstreamPolicyWithoutHTTPRoute(t *testing.T) { defer resp.Body.Close() var ( - config file.Content - buff bytes.Buffer + configDump configDumpResponse + config file.Content + buff bytes.Buffer ) - if err := gojson.NewDecoder(io.TeeReader(resp.Body, &buff)).Decode(&config); err != nil { + if err := gojson.NewDecoder(io.TeeReader(resp.Body, &buff)).Decode(&configDump); err != nil { t.Logf("WARNING: error while decoding config: %+v, response: %s", err, buff.String()) return false } + config = configDump.Config + if len(config.Upstreams) != 1 { t.Logf("WARNING: expected 1 upstream in config: %+v", config) return false diff --git a/test/envtest/telemetry_test.go b/test/envtest/telemetry_test.go index 90b7dfa307..a3aa8d31c4 100644 --- a/test/envtest/telemetry_test.go +++ b/test/envtest/telemetry_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/client-go/rest" gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager" - "github.com/kong/kubernetes-ingress-controller/v3/internal/util" "github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate" ) @@ -68,7 +68,7 @@ func TestTelemetry(t *testing.T) { if !assert.NoError(t, err) { return } - err = manager.Run(ctx, &cfg, util.ConfigDumpDiagnostic{}, logger) + err = manager.Run(ctx, &cfg, diagnostics.ConfigDumpDiagnostic{}, logger) assert.NoError(t, err) }() From 1a1cf18833e9926c93cc80a204fe2543a36d2dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 4 Jun 2024 12:26:31 +0200 Subject: [PATCH 27/48] tests: use expressions router by default in test manifests (#6119) * tests: use expressions router by default in test manifests * tests: extend the envtest suite timeout to 8m --- Makefile | 2 +- config/components/konnect/manager.yaml | 15 --------------- config/variants/konnect/base/manager.yaml | 15 --------------- config/variants/konnect/debug/manager_debug.yaml | 14 -------------- .../all-in-one-dbless-konnect-enterprise.yaml | 4 ++-- test/e2e/manifests/all-in-one-dbless-konnect.yaml | 4 ++-- 6 files changed, 5 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index b80a59117e..926ee7b1f5 100644 --- a/Makefile +++ b/Makefile @@ -430,7 +430,7 @@ test.golden.update: use-setup-envtest: $(SETUP_ENVTEST) use -ENVTEST_TIMEOUT ?= 5m +ENVTEST_TIMEOUT ?= 8m .PHONY: _test.envtest .ONESHELL: _test.envtest diff --git a/config/components/konnect/manager.yaml b/config/components/konnect/manager.yaml index eb6a4af5af..a9e4bf260b 100644 --- a/config/components/konnect/manager.yaml +++ b/config/components/konnect/manager.yaml @@ -27,18 +27,3 @@ spec: secretKeyRef: name: konnect-client-tls key: tls.key ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: proxy-kong - namespace: kong -spec: - template: - spec: - containers: - - name: proxy - env: - - name: KONG_ROUTER_FLAVOR - value: traditional_compatible - diff --git a/config/variants/konnect/base/manager.yaml b/config/variants/konnect/base/manager.yaml index eb6a4af5af..a9e4bf260b 100644 --- a/config/variants/konnect/base/manager.yaml +++ b/config/variants/konnect/base/manager.yaml @@ -27,18 +27,3 @@ spec: secretKeyRef: name: konnect-client-tls key: tls.key ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: proxy-kong - namespace: kong -spec: - template: - spec: - containers: - - name: proxy - env: - - name: KONG_ROUTER_FLAVOR - value: traditional_compatible - diff --git a/config/variants/konnect/debug/manager_debug.yaml b/config/variants/konnect/debug/manager_debug.yaml index b2a3281ada..59a6c00701 100644 --- a/config/variants/konnect/debug/manager_debug.yaml +++ b/config/variants/konnect/debug/manager_debug.yaml @@ -30,17 +30,3 @@ spec: - name: CONTROLLER_KONNECT_ADDRESS value: https://us.kic.api.konghq.tech image: kic-placeholder:placeholder ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: proxy-kong - namespace: kong -spec: - template: - spec: - containers: - - name: proxy - env: - - name: KONG_ROUTER_FLAVOR - value: traditional_compatible diff --git a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml index 6e7c6b15be..a9ec331080 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml @@ -3648,8 +3648,6 @@ spec: automountServiceAccountToken: false containers: - env: - - name: KONG_ROUTER_FLAVOR - value: traditional_compatible - name: KONG_PROXY_LISTEN value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384 @@ -3671,6 +3669,8 @@ spec: value: /dev/stderr - name: KONG_PROXY_ERROR_LOG value: /dev/stderr + - name: KONG_ROUTER_FLAVOR + value: expressions image: kong/kong-gateway:3.5 lifecycle: preStop: diff --git a/test/e2e/manifests/all-in-one-dbless-konnect.yaml b/test/e2e/manifests/all-in-one-dbless-konnect.yaml index 8d38ba4f79..982c678d78 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect.yaml @@ -3650,8 +3650,6 @@ spec: automountServiceAccountToken: false containers: - env: - - name: KONG_ROUTER_FLAVOR - value: traditional_compatible - name: KONG_PROXY_LISTEN value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384 @@ -3673,6 +3671,8 @@ spec: value: /dev/stderr - name: KONG_PROXY_ERROR_LOG value: /dev/stderr + - name: KONG_ROUTER_FLAVOR + value: expressions image: kong:3.6 lifecycle: preStop: From d9968adfb8bd6d1946511b9c4968a0a221d2b879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 4 Jun 2024 17:56:01 +0200 Subject: [PATCH 28/48] tests: skip flaky fallback config envtest (#6128) --- test/envtest/metrics_envtest_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/envtest/metrics_envtest_test.go b/test/envtest/metrics_envtest_test.go index db0793ddd7..e2276ac6ed 100644 --- a/test/envtest/metrics_envtest_test.go +++ b/test/envtest/metrics_envtest_test.go @@ -36,9 +36,11 @@ func TestMetricsAreServed(t *testing.T) { withPushError bool fallbackConfigurationEnabled bool expectedMetrics []string + skippedMessage string }{ { name: "with push error and FallbackConfiguration enabled", + skippedMessage: "flaky, see https://github.com/Kong/kubernetes-ingress-controller/issues/6125", withPushError: true, fallbackConfigurationEnabled: true, expectedMetrics: []string{ @@ -74,6 +76,9 @@ func TestMetricsAreServed(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.skippedMessage != "" { + t.Skip(tc.skippedMessage) + } ctx, cancel := context.WithTimeout(context.Background(), waitTime) defer cancel() From bb77d09d5e54dcd96b3616a04e7eabe609ceac02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 4 Jun 2024 19:36:16 +0200 Subject: [PATCH 29/48] tests: fix flaky TestCustomVault test (#6129) --- test/integration/vault_test.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/test/integration/vault_test.go b/test/integration/vault_test.go index 40a1b8fefd..f3c96962b1 100644 --- a/test/integration/vault_test.go +++ b/test/integration/vault_test.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kong/kubernetes-ingress-controller/v3/internal/annotations" dpconf "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/config" @@ -114,11 +115,23 @@ func TestCustomVault(t *testing.T) { require.NoError(t, err) t.Logf("attach plugin to ingress and check if the config from vault takes effect") - ingress, err = env.Cluster().Client().NetworkingV1().Ingresses(ns.Name).Get(ctx, ingress.Name, metav1.GetOptions{}) - require.NoError(t, err) - ingress.Annotations[annotations.AnnotationPrefix+annotations.PluginsKey] = "request-transformer-advanced" - _, err = env.Cluster().Client().NetworkingV1().Ingresses(ns.Name).Update(ctx, ingress, metav1.UpdateOptions{}) - require.NoError(t, err) + ingressName := ingress.Name + require.Eventually(t, func() bool { + ingClient := env.Cluster().Client().NetworkingV1().Ingresses(ns.Name) + ingress, err = ingClient.Get(ctx, ingressName, metav1.GetOptions{}) + if err != nil { + t.Logf("error getting %s: %v", client.ObjectKeyFromObject(ingress), err) + return false + } + ingress.Annotations[annotations.AnnotationPrefix+annotations.PluginsKey] = "request-transformer-advanced" + _, err = ingClient.Update(ctx, ingress, metav1.UpdateOptions{}) + if err != nil { + t.Logf("error annotating %s: %v", client.ObjectKeyFromObject(ingress), err) + return false + } + return true + }, ingressWait, waitTick) + require.Eventuallyf(t, func() bool { resp, err := helpers.DefaultHTTPClient().Get(fmt.Sprintf("%s/test_custom_vault/headers", proxyHTTPURL)) if err != nil { From 4d6074ce5214fdade3afb14898c6eafd83dacac9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 21:15:10 +0200 Subject: [PATCH 30/48] chore(deps): update kong docker tag (#6120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update kong docker tag * ci: install mise in regenerate GitHub workflow * chore: regenerate manifests * ci: use token for commit pushing after regenerating * chore: add git status to regenerate workflow --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Patryk Małek Co-authored-by: github-actions --- .github/test_dependencies.yaml | 4 ++-- .github/workflows/regenerate_on_deps_bump.yaml | 13 +++++++++++-- config/image/oss/kustomization.yaml | 2 +- test/e2e/manifests/all-in-one-dbless-konnect.yaml | 2 +- test/e2e/manifests/all-in-one-dbless.yaml | 2 +- .../all-in-one-postgres-multiple-gateways.yaml | 8 ++++---- test/e2e/manifests/all-in-one-postgres.yaml | 8 ++++---- 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/test_dependencies.yaml b/.github/test_dependencies.yaml index 6614ee4a1f..f7821c878a 100644 --- a/.github/test_dependencies.yaml +++ b/.github/test_dependencies.yaml @@ -44,13 +44,13 @@ integration: # renovate: datasource=docker depName=kindest/node versioning=docker kind: 'v1.30.0' # renovate: datasource=docker depName=kong versioning=docker - kong-oss: '3.6.1' + kong-oss: '3.7.0' # renovate: datasource=docker depName=kong/kong-gateway versioning=docker kong-ee: '3.7.0.0' kongintegration: # renovate: datasource=docker depName=kong versioning=docker - kong-oss: '3.6.1' + kong-oss: '3.7.0' envtests: # Because of a bug that was introduced in Kong EE 3.5 (https://konghq.atlassian.net/browse/KAG-3699), diff --git a/.github/workflows/regenerate_on_deps_bump.yaml b/.github/workflows/regenerate_on_deps_bump.yaml index b011ec1f5c..df1491f4f9 100644 --- a/.github/workflows/regenerate_on_deps_bump.yaml +++ b/.github/workflows/regenerate_on_deps_bump.yaml @@ -22,16 +22,25 @@ jobs: with: go-version-file: go.mod + - uses: jdx/mise-action@v2 + with: + install: false + - name: regenerate run: make manifests + - name: Set github url and credentials + run: | + /usr/bin/git config --global --add url."https://${{ secrets.K8S_TEAM_BOT_GH_PAT }}:x-oauth-basic@github".insteadOf ssh://git@github + /usr/bin/git config --global --add url."https://${{ secrets.K8S_TEAM_BOT_GH_PAT }}:x-oauth-basic@github".insteadOf https://github + /usr/bin/git config --global --add url."https://${{ secrets.K8S_TEAM_BOT_GH_PAT }}:x-oauth-basic@github".insteadOf git@github + - name: commit and push (if changes detected) - env: - GITHUB_TOKEN: ${{ secrets.K8S_TEAM_BOT_GH_PAT }} run: | git config --global user.name "github-actions" git config --global user.email "github-actions@users.noreply.github.com" git add ./test/e2e/manifests + git status git diff-index --quiet HEAD || \ git commit -m "chore: regenerate manifests" && \ git push origin ${{ github.event.pull_request.head.ref }} diff --git a/config/image/oss/kustomization.yaml b/config/image/oss/kustomization.yaml index 677a1f2471..5316d15907 100644 --- a/config/image/oss/kustomization.yaml +++ b/config/image/oss/kustomization.yaml @@ -4,7 +4,7 @@ kind: Component images: - name: kong-placeholder newName: kong - newTag: '3.6' + newTag: '3.7' - name: kic-placeholder newName: kong/kubernetes-ingress-controller newTag: '3.1' diff --git a/test/e2e/manifests/all-in-one-dbless-konnect.yaml b/test/e2e/manifests/all-in-one-dbless-konnect.yaml index 982c678d78..ffabcb9190 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect.yaml @@ -3673,7 +3673,7 @@ spec: value: /dev/stderr - name: KONG_ROUTER_FLAVOR value: expressions - image: kong:3.6 + image: kong:3.7 lifecycle: preStop: exec: diff --git a/test/e2e/manifests/all-in-one-dbless.yaml b/test/e2e/manifests/all-in-one-dbless.yaml index b4c71ecaf1..f3c479de2e 100644 --- a/test/e2e/manifests/all-in-one-dbless.yaml +++ b/test/e2e/manifests/all-in-one-dbless.yaml @@ -3658,7 +3658,7 @@ spec: value: /dev/stderr - name: KONG_ROUTER_FLAVOR value: expressions - image: kong:3.6 + image: kong:3.7 lifecycle: preStop: exec: diff --git a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml index b2b10afe5c..2069d66e1e 100644 --- a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml +++ b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml @@ -3616,7 +3616,7 @@ spec: value: postgres - name: KONG_PG_PASSWORD value: kong - image: kong:3.6 + image: kong:3.7 name: wait-for-migrations serviceAccountName: kong-serviceaccount volumes: @@ -3689,7 +3689,7 @@ spec: value: /dev/stderr - name: KONG_ROUTER_FLAVOR value: traditional - image: kong:3.6 + image: kong:3.7 lifecycle: preStop: exec: @@ -3815,7 +3815,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong:3.6 + image: kong:3.7 name: kong-migrations initContainers: - command: @@ -3828,7 +3828,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong:3.6 + image: kong:3.7 name: wait-for-postgres restartPolicy: OnFailure --- diff --git a/test/e2e/manifests/all-in-one-postgres.yaml b/test/e2e/manifests/all-in-one-postgres.yaml index 251a7ca49d..ce07f1a498 100644 --- a/test/e2e/manifests/all-in-one-postgres.yaml +++ b/test/e2e/manifests/all-in-one-postgres.yaml @@ -3565,7 +3565,7 @@ spec: value: /dev/stderr - name: KONG_PROXY_ERROR_LOG value: /dev/stderr - image: kong:3.6 + image: kong:3.7 lifecycle: preStop: exec: @@ -3666,7 +3666,7 @@ spec: value: postgres - name: KONG_PG_PASSWORD value: kong - image: kong:3.6 + image: kong:3.7 name: wait-for-migrations serviceAccountName: kong-serviceaccount volumes: @@ -3755,7 +3755,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong:3.6 + image: kong:3.7 name: kong-migrations initContainers: - command: @@ -3768,7 +3768,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong:3.6 + image: kong:3.7 name: wait-for-postgres restartPolicy: OnFailure --- From bc75c4f23585fae0199a8dd3df7f9221c247ce3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 4 Jun 2024 21:15:49 +0200 Subject: [PATCH 31/48] chore: fix dev admisison webhook config (#6127) --- .../manager_dev_webhook/manager_webhook_secret.yaml | 4 ++-- .../validating_webhook_configuration.yaml | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/components/manager_dev_webhook/manager_webhook_secret.yaml b/config/components/manager_dev_webhook/manager_webhook_secret.yaml index 9ee4b4018c..744290927d 100644 --- a/config/components/manager_dev_webhook/manager_webhook_secret.yaml +++ b/config/components/manager_dev_webhook/manager_webhook_secret.yaml @@ -2,8 +2,8 @@ # This is provided for ease of use and is not meant to be used in production. apiVersion: v1 data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWRENDQWp5Z0F3SUJBZ0lVTUw1UEVFakNkdkg1VjdrQjRueXIySVNISVFRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0t6RXBNQ2NHQTFVRUF3d2dhMjl1WnkxMllXeHBaR0YwYVc5dUxYZGxZbWh2YjJzdWEyOXVaeTV6ZG1NdwpIaGNOTWpNeE1URTFNVEl5TXpNeldoY05NalF4TVRFME1USXlNek16V2pBck1Ta3dKd1lEVlFRRERDQnJiMjVuCkxYWmhiR2xrWVhScGIyNHRkMlZpYUc5dmF5NXJiMjVuTG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQUQKZ2dFUEFEQ0NBUW9DZ2dFQkFJbmc4VEdvL0ZGU3NoWitidWltY1Y1Vk9xMnYrT2xGRFFlMGVUSEp4Uk04QUtodAo0NmNTMkpQVjA1MFlLc3JzajN1M3EwMU1wZ3NzdENsS1ptWk0rYjhjcllyOUhhcE5GTmx1TUxCb1BDR1ZBcUcxCjBQVWJFTzRhUUxoeDhiQTJ4OFFYSkhLWEpzVVFqYzF3MkhmV2xPUXNJYzJBU3Vqd2t5TWJGcUNQNHRramhXWWEKU3VYNktOOXZ5RG1HZG5zWWc2dm02dEJmeWVkUWdMdmlXQ3FPN0VEMjJTd2tycFV1bWg2dzJWQ2FwTnlTQW82RAozT0QrWUFyT2p4ajF1L3g2N28xSXVnVVBFa25zakFmcTVGWGNEcWlYUldGU2tGOThVL1BEQ2V3RGp4WXgyWHhmCkJ3OFRnaW8zekhJaThOeStkQmpDOHRpQzBPVW4xcjNvREhDVytvOENBd0VBQWFOd01HNHdLd1lEVlIwUkJDUXcKSW9JZ2EyOXVaeTEyWVd4cFpHRjBhVzl1TFhkbFltaHZiMnN1YTI5dVp5NXpkbU13Q3dZRFZSMFBCQVFEQWdlQQpNQk1HQTFVZEpRUU1NQW9HQ0NzR0FRVUZCd01CTUIwR0ExVWREZ1FXQkJTWnF3dHFzZFVoZ3F2b2ViNnRmelN0Cm5FUk1EakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBWk83eDd3YU51TDZZRlFoT09pQUFJL2VGdGNqZGFEbisKcENqdWRtSEpUNllTK2l2NitWemVDMEZ2cmVqdGhlak1pV3orYTVLaUE2d2tZUXBvRzNYVkQ4azlCR25hMmFaSQpBaWtGT0MvM09uWHpwd0FZcFNNVTRIM3cvK0M1bGY3eVdoUE1jMExHNDRNMm96SHhhR21paDB6ZHpEcitObnF2CjJwdG8vVW9BelBXcVBrV2FnTUliRzVHSkFFZCtvaUthRU9xSVQ3R0lsbkhzQTI4STlBQThzMzNqL25XQ2VyY0sKZ3NqeWR5SVFmTVljY0c1L2E0T0FHeUFTcXd3TGo2Mys1TzB1V29JYzYwcy9aemZ3bkt0VDlLNmN5Ymh4T25ZZwpmVEhCdkx0emNxQjZZTldqUTk2ZFNmak4vVlkzLzZnaG5wYWVSSTNPWjVzWEU4MWpNSU9SNlE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ0o0UEV4cVB4UlVySVcKZm03b3BuRmVWVHF0ci9qcFJRMEh0SGt4eWNVVFBBQ29iZU9uRXRpVDFkT2RHQ3JLN0k5N3Q2dE5US1lMTExRcApTbVptVFBtL0hLMksvUjJxVFJUWmJqQ3dhRHdobFFLaHRkRDFHeER1R2tDNGNmR3dOc2ZFRnlSeWx5YkZFSTNOCmNOaDMxcFRrTENITmdFcm84Sk1qR3hhZ2orTFpJNFZtR2tybCtpamZiOGc1aG5aN0dJT3I1dXJRWDhublVJQzcKNGxncWp1eEE5dGtzSks2Vkxwb2VzTmxRbXFUY2tnS09nOXpnL21BS3pvOFk5YnY4ZXU2TlNMb0ZEeEpKN0l3SAo2dVJWM0E2b2wwVmhVcEJmZkZQend3bnNBNDhXTWRsOFh3Y1BFNElxTjh4eUl2RGN2blFZd3ZMWWd0RGxKOWE5CjZBeHdsdnFQQWdNQkFBRUNnZ0VBQlRodTc5bzNoUmNrUnFpT2RIcWRwWFllcWFZVmNnRTdHbmd1UVBRa01lYlgKNXJ2ZzBjQnJoYWlqOUxsYTdXN1BJL2cvdXNhSCtoL3dML1drTW9QTXpwWFYvUnc1aUxnd2JiUHE5NSs5NitoagpjYmhvYXpYSWlvVDZ2NDNmVFdlNytwZFJ1aXFZSTh6TlFhOTVoZkdxTUZWNm92aEhrUVVjZHE4M1NFOEtYLzNqCmZCakFtbG5weVFLZEZKRFBZSTQwaFdkSlppd05HNnl1WVd0RjR2Zjk5Y3hHaHBreFhYZVRRZ2hTcGNVbFNrVzgKK2M0VVNxeU9rem10bFUxRzh1dzdIK0h0QmY4dlZwVDR5U1oxaHFEeWhBSkdxcDIrN1g2MTZDZmw0STYxckZEeQpZMG5ZdWxFTmsvNWhZSmllZkhFUlhRNE5aUGFLK3UyTW1EbkhDZGNpdVFLQmdRQytYZlZ1N1FsL2VCaWNyeXdQCnVDMmR5SjdiSUgwb1pDclhqUlRIS1JZTHZYajFtdTE5U0RjQVIzenAvL1UyWWpscW56SGc3VUJNVnlZWmxCN0oKb0ZMcGJwNnJIME9kK0llcisvZGl5MnBjTnZaMmZjVkdJTjZsZmk1T1FVWmJxOGt4K1NhRDZOcmJZeWQwL3JNWAp2NmppaEVldDQ0ZTVUTnljc2Vsa0xZRVl0UUtCZ1FDNWFrczA2YTRRTlM0NVRoMVZCU2NwSEdUdFkxbXkyaU9jCnl0bEdmbXV0ekNqSWwzU1BkMEJjeVFlVnpCamo5cGtoVVpKRXEyQXdmTVIxTmlNZTAwRktpak55SWRaZ1RvNVAKaEZmV3hNOE1mamNvSTlMMFhSdTdQc2NqS0svNHV6VWtjZUtTY1VtNTY3Rk9mNWNsZmh1WDF3WGhWalpneTVpVworNWdQa254a3N3S0JnUUM4eTZweGpKdnkwMFIxZ0RVT2tmYUxtVUFTeWpIV01TRmNEUXNpU2RrWFk1M20xdlBaCllCbE1LWm4wNkdoa3V4MStaTXV1Nnh6dG1UQ3NCWDVUTUxHSjJLOTd2dEhzaFdMb2FrZDZyNHFZVWRvMHdaODQKWWJqdUlDb0VhakJCRWluRGFmbU1zUTc4cldXZ1hrbDNzQmpxTFk1NUlrS2t2MW03L2FZZU9CTGtVUUtCZ0VoQwpnWjdVZDA2L3V3MEFRWFF4OXVvUnM4L0VTVi9ubmJ0c1hyTVhiOVdpM0Q0WXNJZDgvU3RyK1RYSy9lUlI1YW5UCmhZS1htM3dxRTlKdVQ4K2lteTUybjhnYUlkY1VwbWVjOXpLdkx0WDZsbnBoUThTU1NNMTNrTnBGOEJhcXR2SkcKSS92WWhOZ2RYOU5zN0RYamFOT0xMREoraStDN1YvTjNoL0tCcjFMN0FvR0FEbzBMVC9XRWhZTnRuWnQrdkFlNwo3SEw0Y3ZrTXlBR3pPdEtSQmNna2kxQTNYWHpBUndUWkNxR2lXeDNTMXRqdURURmxHUzRONDJjdDN1VVRmRVpSCjZCZ2MzU1RqVGMvalNFbzFuZis0SnhlanJFWHFHV1dyOWJEY3pmUFcxS1lwdDF3UktHcXh3MUdUazBqL0FxZTUKMjltbFMzYlJ6T0c1NlBPZThCSWYrMjA9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdPRENDQkNDZ0F3SUJBZ0lVR2c0dEhBeGJWZmFDaCtob0FWMlUyK2VyMG93d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2VURUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2tOaGJHbG1iM0p1YVdFeEZqQVVCZ05WQkFjTQpEVk5oYmlCR2NtRnVZMmx6WTI4eERUQUxCZ05WQkFvTUJFdHZibWN4R0RBV0JnTlZCQXNNRDB0dmJtZGZTM1ZpClpYSnVaWFJsY3pFVU1CSUdBMVVFQXd3TFkyRXVhMjl1Wnk1d2IyUXdIaGNOTWpRd05qQTBNVE13TURNd1doY04KTXpRd05qQXlNVE13TURNd1dqQ0JrREVMTUFrR0ExVUVCaE1DVlZNeEV6QVJCZ05WQkFnTUNrTmhiR2xtYjNKdQphV0V4RmpBVUJnTlZCQWNNRFZOaGJpQkdjbUZ1WTJselkyOHhEVEFMQmdOVkJBb01CRXR2Ym1jeEdEQVdCZ05WCkJBc01EMHR2Ym1kZlMzVmlaWEp1WlhSbGN6RXJNQ2tHQTFVRUF3d2lLaTVyYjI1bkxYWmhiR2xrWVhScGIyNHQKZDJWaWFHOXZheTVyYjI1bkxuTjJZekNDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQgpBTVkyeGR2ZEZyUXFoOXRzenNFd1FHaFBXQUNzTXFiYzdkVXF3dWVSNktTKysvTVZZL1B4cDVDYXl1MmNhaDl3Ci82eW5zc0hXeUlOSXRmM2NKL3I5MGZFSE1ONVdUNDIzVFA2Z3o4cWFBcGZGZlB1OCtxOG05ZUtPMGpWclV2VnUKQ2dFU3BWSVBPMFNxM3JpM0xZNnFZalJwRzQ2bkFwTEJ1cGQ1WC9qWXJ1c25XWmlWNFdtSVQzZWs2ZURsbVhhRwpMRExlQlBscG8vWktCNGFqZUk4Umt5ckRtY0FHcmRDSUs4ek5vWmNkdCtJNjg5NmhGbmQ4WDc2SkxWL3cvVW9RCkZZaE80Q1NFWStaOGRDeWI2dEZjaWk5cWhsTmdBSDYrYWtrbGkvbVpaOHBzWjVFUVExMi9HWS8yNzIxaEtxMU4KczhldTlJNERGTEptSHhkSE1nWGY0eHNqMmRnTlZKL0NDd2szdDBtOHVzTUVESXZnZ3gycmluTitEbDhkY2cyTApSM3pna1VERlQydkpneSt1aDI0VWIwMVdHekd5R0dJSUQ4cGltSzlQbmxkdkpEU3pKWG5OaXNKSGE3ZVRXVnlDCjlPbEp0cTVWL2x0Y21WakYxWFFrbnhWZTcwcHZCcGJ4aFZTcjhxQmJTMTBNTk9EdkNkYmFvZXBxMFhkK2E3SEcKSDFJVXVEQk5pdWVJWnF3YzFQaWVFNlZMa1B5bHdlMlBGUUlnUE5lQkRZRURkbXlHbVJXWmlPUmF6S3J5c2NRWQpUaUxZNE54cklJOFcwVml1cE9MQkRZWTVpM2tHaDZSN2VpNEJZSlZjbFYvcDludjAwZG9UbnFRM3pPZ1ZrVitvClU0OGRTeVRFZlFDVk1OSWlBRll6UVlnbXRsOWZRcUJ2WTdzUDZBYmtJQ2laQWdNQkFBR2pnWjh3Z1p3d0h3WUQKVlIwakJCZ3dGb0FVNUxFemVWNCs0bk84SlZKd2VvUDBLRzlWcUU4d0NRWURWUjBUQkFJd0FEQlBCZ05WSFJFRQpTREJHZ2lJcUxtdHZibWN0ZG1Gc2FXUmhkR2x2YmkxM1pXSm9iMjlyTG10dmJtY3VjM1pqZ2lCcmIyNW5MWFpoCmJHbGtZWFJwYjI0dGQyVmlhRzl2YXk1cmIyNW5Mbk4yWXpBZEJnTlZIUTRFRmdRVXZJZlZMRitFMmFIYWgxS3gKbThxelIvOGFtRmt3RFFZSktvWklodmNOQVFFTEJRQURnZ0lCQUZqQVNNUlZiQ1M2UXBQaFlVRDRrbHlNRHY3SApMREVhT0RtOTAvUWI4WEczbno1bGp0aFlyMnMyQitkNFdCbjlWNUhBMDRhWmpkVytNcjVNMWRXQmE3cVJKdDRWCnRFOUJyTTlxVDg2aXRiLzZOVzRqVDI4ckhUdHVpQlRDNW9OQjBLbytUMCtFRW85VEY5OHRnTE16RkVjWGp1R2QKOG5Ub0J6dnpxbmF2N1ZEVU5xQ3gxZElWTkVOQlpmTnVSTUM0aWJmNG9ONlBOOVJuSGtzUjh6ZjhKSUdNTkN2NwpMdnBGM0Z4Y0YxMmRBRnFVQWNQa0NSc1k0K3FsNzQ1WlpTeWcvUUtTWUFodWt4MUhHQWhYdmVjbmo5UHI5dVROCndBcXFwblBrVHFNR0REZUM5RGpQVmVJQkM5bnBBY0YrYjJIMW83MFYrYmRnMklwSFAydnExVmJpcFpRa3ZEMVIKTWRVK0dYOUNUbzlERGFFVnFjSEsyYWJqWWhRV2x4Z1Vsb2l0WVptZUI3NzlwNEpwb1FIbjhPTW0wSW9hanA2ZAo0ZkdISzRlNDF1Rmg4bFl0NFVxdnZNZElYSmdURHVTVjRDemVhSGM2aHBLd25yM0lKQTNmV2FMWGl2RGJ1SzB4CmtXSUtSSU5SaC9IYWo5VlNqejhzV3pNL1M2MTlqYmNvRElvelQ4aXU0OWV2aW9KS2JiRUprMkhHVnl1Qk02MHQKM3RBZ2VxbVh0UGI4WWpFcmV1S0Z2Y09WRXZnZHd3MHlXaEk1RHZTck1jM3J4T2RGaFQrMjNPcDlyZmJCL1duVQpUTHRQVkhZTGRNeGxrMVFxMHBUbGJiYTdQRkVWRXNKdjJMK0MzUlFIdS84Qk1YWHJjZ2lLaXNLODkvNzdvcElUCmo5UHF2RW5NdDZraHZWNGMKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRREdOc1hiM1JhMEtvZmIKYk03Qk1FQm9UMWdBckRLbTNPM1ZLc0xua2Vpa3Z2dnpGV1B6OGFlUW1zcnRuR29mY1Arc3A3TEIxc2lEU0xYOQozQ2Y2L2RIeEJ6RGVWaytOdDB6K29NL0ttZ0tYeFh6N3ZQcXZKdlhpanRJMWExTDFiZ29CRXFWU0R6dEVxdDY0CnR5Mk9xbUkwYVJ1T3B3S1N3YnFYZVYvNDJLN3JKMW1ZbGVGcGlFOTNwT25nNVpsMmhpd3kzZ1Q1YWFQMlNnZUcKbzNpUEVaTXF3NW5BQnEzUWlDdk16YUdYSGJmaU92UGVvUlozZkYrK2lTMWY4UDFLRUJXSVR1QWtoR1BtZkhRcwptK3JSWElvdmFvWlRZQUIrdm1wSkpZdjVtV2ZLYkdlUkVFTmR2eG1QOXU5dFlTcXRUYlBIcnZTT0F4U3laaDhYClJ6SUYzK01iSTluWURWU2Z3Z3NKTjdkSnZMckRCQXlMNElNZHE0cHpmZzVmSFhJTmkwZDg0SkZBeFU5cnlZTXYKcm9kdUZHOU5WaHN4c2hoaUNBL0tZcGl2VDU1WGJ5UTBzeVY1ellyQ1IydTNrMWxjZ3ZUcFNiYXVWZjViWEpsWQp4ZFYwSko4Vlh1OUtid2FXOFlWVXEvS2dXMHRkRERUZzd3blcycUhxYXRGM2ZtdXh4aDlTRkxnd1RZcm5pR2FzCkhOVDRuaE9sUzVEOHBjSHRqeFVDSUR6WGdRMkJBM1pzaHBrVm1ZamtXc3lxOHJIRUdFNGkyT0RjYXlDUEZ0RlkKcnFUaXdRMkdPWXQ1Qm9la2Uzb3VBV0NWWEpWZjZmWjc5TkhhRTU2a044em9GWkZmcUZPUEhVc2t4SDBBbFREUwpJZ0JXTTBHSUpyWmZYMEtnYjJPN0QrZ0c1Q0FvbVFJREFRQUJBb0lDQUNZUUZKU3c4eEFyN3FUVFV5ekVBRXBICm1rV2V5NmRCVzZhSXJKN2RaUWhsNkduNG5KWVd2Sy9IR2RyaEkzdmdMaWpRbDBJajVhTllUaUp1cVhySVRRMTAKN1F4MUZKZkpNRTdoUUtYT1Ixc3Z3RjF6SDh5TXRjRUkzaE1HQmdzOVUweDdjU1c5NlFxNXFVVmRYN0U2eE5yOQpxQjRQc2kzT1orOTRqbFp4L2IwSWRHc1p0UXlYOFhLdlIzNDVlT3ZLdmlnU2ZIQU11a0NzWEorZE9xUjJvb1VlCnhLS2NaYnBhdTlaN1VtRjdnbmsrQ3VDZ25IamJlR05WWWxJL0s5U1NIQ0J0elJzV0ZUMElmeUtsMmtUVCtYRTcKaW9udU1Jb21OTERSTmZ6N1dYUUV0eXFqTGR5TnV2SHJZSFRucU5OWXpZSTE5WjVqdHhDL0RZOWh5NWhYTXZ3SApybjY0YWRHM3pmRVNzcHVwa3ViQnJFUng0TkNYRm9zOCtrRVNoVHUwQks0Q3ZNL1l1ZTdHbjZET2JnbjNRdjdDCjBKSkx5aDE5SW9QZ3RSaWoyMlg4NWMrdksrQ2JVM1MybEQwcmE2SE5sV2J1U1lBK1RtN0VVTFRUazh0L3lOOTcKbHF2NS8vQnNiZUl3UHJveW01cHhWcHpIN2pOdFF5RWw0T0RIa2d1LzZRNmh0cFpEdWJxQ2UxK3Z0Sjc0NStqOApXNEtabUc3bW5kOXFDT3RWbzcwSnBWazk0eUM0Z0wrSXR1ZXpwcmhjMHNuRzlVSGk3RG1abXQzY1FqTUNQaEpMCm0rRGowbnRJblZJYWg5NjBEd1BjV2E2TzFkT3ZaRy9lek9QNVJnRDNPQThUQnZGN3BMdS9YUTk1Q1cwblJXS2cKSnJqZHVUWS9lcFpwUlhNMVhybFZBb0lCQVFEbkRIbWE1N2JycWkzZXhtRXhZOEN0aFk0empRbFY1QWxYWTAxTApNVThrMDZGcm9LY29ocEhUeVFnZTV2V01tNmFpUkhoNWFTcFpqaUgvSFk5SDNDdS9jK1pNbzF5UFNGcnIwaVBGCkFvdDVleTdzbDBQTzJwc0ZpSDB5dnVIUytmNGU3VzRJbm1XZHpuZVhWcXlmMys3SUMwZFhqYnhIeFlwUHkwU28KTFRQZisvaXhVSDQ5SzdIcEdwUzFsaTZKeld1R2dkaTcrVGRQQ0Qwa0djenNIazUrS1V3amg2d2Voais1L3lJUgplTFBzR09Hd1VyS012T1dITElnWFRaV0tWSE5XcWcyNTlMY3BlUytvM0Q5ZDZkbkhaOFYweTYxRUN1ejJxWXJPClpvbk9pY3BVVDVlK0lxaVBuYThVZktHSTdWb0sxNVFWUWVCYVg2YjcvRkNxY2RIbkFvSUJBUURibm8zM2pQeFYKRi9ScWE4ajltNktlWk5ydDJJM2JrN0dVbGF0TnlCenhkYlV2aEc1MDhKR0lQWE9KT094em9OWlRnNkFoL3I5dgoycUhMR2h2TTJTTzZCcjhkTDNoemZnWSs5YlcrSytsU0VoNExWeVE3SVNBeCtTMVhveHhUVGlwWGF0cHB5UkpNClo4aHVXNzZuWXh3bDZjNVkxVEZ0cFFSeUlISkF6Qi9YQnpTaXRETjRxZkNqckUzYk9pMnBUTUh1ZHZjT3c3ZlYKK1A4cCt2QzhpY0tOWTdBT01lME8yQlpvK2ZQVE11dHg1VUpERklZRFNXNXBOWG13WVdLNTdOVVlRT25FK0FDdgpBNXBlbEV5am15YUt3Y0NFblE4Ymg4VWFNRTVucDBmbTZIVlVLUGFscjFMS3dDTFR1YytLK0tvK1F0QlRJbjlzCkZPYzZMNjhXeStGL0FvSUJBUUNzQWZlT1FTOUc1eHpiR3VsRXNiVEIrZ25SaXhBR0o0eGt5SUxFbGVNTDBabjgKM0U5VnRrbGVWKzE1eEF2T01CcXY5eldSZlorUHFHYmEzSkRNdUxiQkEzSFNZRlFLUDUyZ3JvTCtxbFJYamtOeQowM0loejFGVm56VkYwQ0dpeFlaUVZBWjAyQ2RpZ2xFNkU4YlVCd3huVlM0NW1rVXZVWHNVeUlsR2d0QjUwY1psCml6MVFJUFdFU3N1bkhEVnRWY2JWRGxuaUp6anIxNEJkSGZBWFlNQ2kzKy9WQzY0eDAxUWlEalM0dVJtSmpVU0gKMWlraTZZWWZTaUhPNTIySzNEQTV0c1FkU25nSm9qUy9DNmtKSzQxOERGOU9Ba3Z0dWd5TDNkQit0SXVuZmFGcApmdy9DOTE1eC9MeFpEaWZjSG9mSVJwSHgrV2NqSU03YURnK250TERGQW9JQkFRQzRudWh2WXppNGZBTys5czhtCnl4QUFvWDRkbGY5aXlCenZjSVpxUThCNUIxK0NDNDBqaHh5QWNGQlEyZWFFS1lBakFySzZBVUtEVUVMVXp5VHgKcHRST3pOOGFOTTdJSC9nMk15NU9LUEhpU1ZLeWE5WU1Vd09TbndzTDhoV2N2a2YvNXRhbk9SM0YxelQ1K093awpJTUFINnkzSkphZUFxY2s3KzZTd2JpaVNCZitzaTFuOXBMYWprUFIrUjhFYzRtYmhCV2NaSlZURWJxWnFid2F6CktBZkIvanlCSWwxTExrSmdpMGI2azRLejQyczVvdVlwbXpCVEIxNDk5UkFlaGtaNU5oQ093WUVwbnhqRlMxdkYKNldhVUhONnZYS3pYa3VJUjZ1dnVYUVNueTJEZWwvVUlRWU9TNThRZlFzT0M2eG1LYjNaYmZOT3JVME15ZWVWeApmNEVYQW9JQkFEZUZmYllLQWNFVzJTTnM5ZFpnRm80QmdyeENWZmZmZWNueG8wd1ZvcGxvWW04SndlUkRRdGFvCmhzVWpVOHFoazVkUlNiMDdrU1p1QkFsSFF5aXZGajc5dEtTUUMxTUZ1WkF3WnhROGoyTDV3MnhRTG1URTB0Z1QKK3oyZUtObUYyNEo2LzJTRGQrSEZKeklsUG54SjFaM3pmeG9TVjdTb1NFNUJCSXlxeDNQSkFRZTVtKzQxMXFrYwpNdGcxSlM3cGpteFJTYnl5aGpKZ042QXFyNFYyVGNXVWxpc2hUeTJjeDRySVZqRVJ5djRLNzMrUVpHRGVFVi9wCi9WS2N0SVRKeFh5K0k0ZVUxODNDK3ZKRDZUSWNpdENxQ1M5MlFicXgwdFZqczJSNnVNZW5ZUEoxeXQvRXdLYi8KcTQ4eGlpd1RjUEdKb3VMUVQyOC9oYkRONGI3OHRFMD0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= kind: Secret metadata: name: kong-validation-webhook diff --git a/config/components/manager_dev_webhook/validating_webhook_configuration.yaml b/config/components/manager_dev_webhook/validating_webhook_configuration.yaml index d7ffe466f5..082a1c65dd 100644 --- a/config/components/manager_dev_webhook/validating_webhook_configuration.yaml +++ b/config/components/manager_dev_webhook/validating_webhook_configuration.yaml @@ -6,11 +6,15 @@ webhooks: - admissionReviewVersions: - v1beta1 clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUYwekNDQTd1Z0F3SUJBZ0lVWU1qRXoyUEMxZmpKYjRORVRpTjFLc1lkTkZNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2VURUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQWdNQ2tOaGJHbG1iM0p1YVdFeEZqQVVCZ05WQkFjTQpEVk5oYmlCR2NtRnVZMmx6WTI4eERUQUxCZ05WQkFvTUJFdHZibWN4R0RBV0JnTlZCQXNNRDB0dmJtZGZTM1ZpClpYSnVaWFJsY3pFVU1CSUdBMVVFQXd3TFkyRXVhMjl1Wnk1d2IyUXdIaGNOTWpRd05qQTBNVE13TURJNVdoY04KTXpRd05qQXlNVE13TURJNVdqQjVNUXN3Q1FZRFZRUUdFd0pWVXpFVE1CRUdBMVVFQ0F3S1EyRnNhV1p2Y201cApZVEVXTUJRR0ExVUVCd3dOVTJGdUlFWnlZVzVqYVhOamJ6RU5NQXNHQTFVRUNnd0VTMjl1WnpFWU1CWUdBMVVFCkN3d1BTMjl1WjE5TGRXSmxjbTVsZEdWek1SUXdFZ1lEVlFRRERBdGpZUzVyYjI1bkxuQnZaRENDQWlJd0RRWUoKS29aSWh2Y05BUUVCQlFBRGdnSVBBRENDQWdvQ2dnSUJBS2pVNkUxSkVVUWtsWjdhYURIQnBQR3J5N3QwazlrVgpTdjBvbk1LSWNKK1NIWVdOdjRJQjBPZENBOTFYNUpOMXl6bzhwQkY0ZjhzQjF5S3BURXNtcS9LenQwdTdGYVJtClNoL2t2UlVkUWxNMUdiQ1RUdzdDOElrd3VrMEV1dHpvMi9DR3RTOWNVbEZOajRLYkNlT0ZLRVFvVldmSllqby8KM0Q0ZlFyZHQwRDdISjgwV0JPNUowNTdyY0NOUGc0MjlvN2R4RTFMVjNoaTEzNFJZcjNJNVNGcXJQZSt3TWNwRwpUOExnTjZYZk80VmprVXZuczZYY28wNkxhMnpvdVdtd2VGdzQvU3REMExxSGk4aW9GVGVxbEhYTjZEN3grL2pQCjVlVVBnUzFmdWNGWll5K1F3NUJmRm5nTFhSK1JyVVNNdXBBekFlZEhxa0syV0pGT3Q1TjRlQWxnYS9SRVRBSFMKUi9NSmJMOFFYZEhOeXpBbWl1ZEZmVTM1ZzUzZ0NjZlBZa1VLeS9iZnBhZWZtL2ZCT1ZneW5odTBWSEZQUTJVcgpJWDJQdFYyUG40WTlyL0Y0UjFMbyt3NDlnQ2hwanNlWW5BS1g3NW4zRHJWSXN4QjBwd2RCc2FHUytxVGJxQnNVClR0NlJkamY3aVB3bGV3VmJUNzB0OWx1R3FVZnN4c2lmZEh0aHZYVkloYVBMTEhMb2lBL0dFUWl3UlBuMGFEZWsKaFNCcU05Q0xOYXpXd0tKbVVIdkNQWWtpenAvZlo5aWxHTTBvRXZBaExNaGwvVjY0VWVWRmhveU4ySllVdHM4QgpGUk9IditnNUZhZ3pXVVVvR3hyYU5aMDdGZThMcTNYVWg5Tk5HSXRSb0Mxb0hkVzJHdmhDemNFMS9DbjlRbUNvCmE2UTJCRXFnaWlDakFnTUJBQUdqVXpCUk1CMEdBMVVkRGdRV0JCVGtzVE41WGo3aWM3d2xVbkI2Zy9Rb2IxV28KVHpBZkJnTlZIU01FR0RBV2dCVGtzVE41WGo3aWM3d2xVbkI2Zy9Rb2IxV29UekFQQmdOVkhSTUJBZjhFQlRBRApBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElDQVFCSmJ4S0J5WnNIUTRVMmIzS0k5K2dsRmloYUhWN0pGcVF6CmNDWHZjeE5IUHhEMjRCYTBoYmdUNHU1WjFDVjNhQnpBbXhwMElZRUZtZVRtR0QzV0hNaFFCcTFiREZrWmhzQnEKUTNheHFwY2s2dTVEWHoraXBtVW5ZdHFGWXBTOStieERWVEhmYzdoWC8rTkVubEU0YnF6czdWbDJJbkpwVnZFNwpBdm9KcGFmemFaVGQwZTUyblMvV3c0aWxIRERrdzZCSWlnR0NxaEhYYWk2R3VXUkF0WnlvUm8xN1crSUxOTUU3ClhNbGxEamFKVmg4N1l0SmZwUTMzNmxBaEszRGwreklNbXBuV0JJK011dXU4TnlnQ3ArZ29qRGphOHFveWR0TUsKQkdNVE02MkdPYVo5WkE0Y1JYREFmdGlpN1BkYk8rcHB4Qm16UzNRaGNQT093SXVhanIreTg1Q0pYWDdZOEdwegpHSjZ6NFhKbXBhQ1V3dzJ5L3ZjY3BDN29wVW4wU29HQ2xTWENuNUdGWWpRV1dtOWhIMTl3UHdEME5IZ2M3dXNRCmVja2dtQk15MDNmVkROYUluMEUxTGt6YTZteE9hOWlTdnc3dHBFeVRDS054eGhNM2R1aXkxR0ZjdE1qdzFDeXkKRkxQWFVMY3hOV21HRkc3YS9Bd1pyb1Z3eGdGYWhHa2YwVzlUTU9taWpLYUlYOVZMVEprVzcrSXpMMnFwUGFGNApaQ3JCN1lRTGorVVJJRUQ0blRDYXA5cVgyVnhJUUdMNW9Ka1hYZVhnQ1RlL005L3ZTMnIzT0p2QVJIajdEbDVvCm9mSzlDUDZwdVY5RXlqM0FtS3VVOHM3cUxQVVIwUkpjbUVFTDhlSlk4eGxIUlZ0YmU4MUdUYmxBSTh1NWJheWUKZ21aOVk2aHZPdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K service: - name: kong-controller-validation-webhook + name: kong-validation-webhook namespace: kong port: 443 - failurePolicy: Ignore + # NOTE: By default Kong's helm charts use Ignore for the failurePolicy + # e.g. https://github.com/Kong/charts/blob/fd9deb6ee34d9b9ac4ab4be2188d4564d0b655e5/charts/kong/values.yaml#L586 + # but setting this to Fail here allows devs to see the errors sooner, during development. + failurePolicy: Fail matchPolicy: Equivalent name: validations.kong.konghq.com rules: From 8a88685739440391cc6c47830f2c7b73a8e58d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 5 Jun 2024 09:44:35 +0200 Subject: [PATCH 32/48] fix: reconcile admin API endpoints on all replicas (#6126) --- CHANGELOG.md | 3 +++ internal/controllers/configuration/kongadminapi_controller.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d736b5b99f..462f5ead62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,6 +207,9 @@ Adding a new version? You'll need three changes: - Fixed KIC clearing Gateway API *Route status of routes that it shouldn't reconcilce, e.g. those attached to Gateways that do not belong to GatewayClass that KIC reconciles. [6079](https://github.com/Kong/kubernetes-ingress-controller/pull/6079) +- Fixed KIC non leaders correctly getting up to date Admin API addresses by not + requiring leader election for the related controller. + [6126](https://github.com/Kong/kubernetes-ingress-controller/pull/6126) ### Changed diff --git a/internal/controllers/configuration/kongadminapi_controller.go b/internal/controllers/configuration/kongadminapi_controller.go index 441876e4c3..92806e516e 100644 --- a/internal/controllers/configuration/kongadminapi_controller.go +++ b/internal/controllers/configuration/kongadminapi_controller.go @@ -71,6 +71,10 @@ func (r *KongAdminAPIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error return r.Log }, CacheSyncTimeout: r.CacheSyncTimeout, + // In order to get up to date Admin API endpoints in all KIC replicas, we need to + // not require leader election so that AdminAPI controller runs in all replicas + // and notifies about the changes regardless of the leader election status. + NeedLeaderElection: lo.ToPtr(false), }). Watches(&discoveryv1.EndpointSlice{}, &handler.EnqueueRequestForObject{}, From 83240a10969d397b2363f38b87e9715fc6364aff Mon Sep 17 00:00:00 2001 From: Tao Yi Date: Wed, 5 Jun 2024 21:34:35 +0800 Subject: [PATCH 33/48] feat(custom_entity): add KCE controller (#6055) * add KCE controller * address comments * re-generate cil argument docs * add feature gate KongCustomEntity and address comments * resolve conflicts and fix envtest * address comments of 3rd review * address comments on 4th reviews * fix typos --- CHANGELOG.md | 12 +- FEATURE_GATES.md | 1 + config/crd/kustomization.yaml | 1 + config/rbac/role.yaml | 16 + docs/cli-arguments.md | 1 + hack/generators/cache-stores/spec.go | 4 + .../generators/controllers/networking/main.go | 17 + .../configuration/zz_generated_controllers.go | 165 +++++++++ internal/controllers/utils/utils.go | 7 + .../configfetcher/kongrawstate_test.go | 3 + internal/dataplane/deckgen/deckgen.go | 11 +- .../dataplane/fallback/graph_dependencies.go | 4 + internal/dataplane/kong_client.go | 8 + internal/dataplane/kongstate/customentity.go | 70 +++- .../dataplane/kongstate/customentity_test.go | 110 ++++++ internal/dataplane/kongstate/kongstate.go | 327 +++++++++++++++- .../dataplane/kongstate/kongstate_test.go | 348 +++++++++++++++++- internal/dataplane/kongstate/plugin.go | 11 + internal/dataplane/sendconfig/inmemory.go | 17 + internal/dataplane/sendconfig/sendconfig.go | 8 +- internal/dataplane/sendconfig/strategy.go | 11 +- internal/dataplane/translator/golden_test.go | 9 +- internal/dataplane/translator/translator.go | 43 +++ .../dataplane/translator/translator_test.go | 9 +- internal/manager/config.go | 2 + internal/manager/controllerdef.go | 14 + .../manager/featuregates/feature_gates.go | 13 + .../featuregates/feature_gates_test.go | 10 +- internal/manager/run.go | 3 +- internal/manager/schema_service.go | 36 ++ internal/store/fake_store.go | 10 + internal/store/store.go | 31 ++ internal/store/zz_generated_cache_stores.go | 10 + .../store/zz_generated_cache_stores_test.go | 5 + .../all-in-one-dbless-k4k8s-enterprise.yaml | 213 +++++++++++ .../all-in-one-dbless-konnect-enterprise.yaml | 213 +++++++++++ .../manifests/all-in-one-dbless-konnect.yaml | 213 +++++++++++ test/e2e/manifests/all-in-one-dbless.yaml | 213 +++++++++++ .../all-in-one-postgres-enterprise.yaml | 213 +++++++++++ ...all-in-one-postgres-multiple-gateways.yaml | 213 +++++++++++ test/e2e/manifests/all-in-one-postgres.yaml | 213 +++++++++++ .../envtest/admission_webhook_envtest_test.go | 6 +- test/envtest/telemetry_test.go | 1 + 43 files changed, 2811 insertions(+), 34 deletions(-) create mode 100644 internal/manager/schema_service.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 462f5ead62..45b12ea1c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,9 +140,19 @@ Adding a new version? You'll need three changes: is more robust - leading and trailing whitespace characters are discarded. [#5977](https://github.com/Kong/kubernetes-ingress-controller/pull/5977) - Added the CRD `KongCustomEntity` to support custom Kong entities that are not - defined in KIC yet. + defined in KIC yet. The current version only supports translating custom + entities into declarative configuration in DBless mode, and cannot apply + custom entities to DB backed Kong gateways. + Feature gate `KongCustomEntity` is required to set to `true` to enabled the + `KongCustomEntity` controller. + **Note**: The IDs of Kong services, routes and consumers referred by custom + entities via `foreign` type fields of custom entities are filled by the `FillID` + method of the corresponding type because the IDs of these entities are required + to fill the `foreign` fields of custom entities. So the `FillIDs` feature gate + is also required when `KongCustomEntity` is enabled. [#5982](https://github.com/Kong/kubernetes-ingress-controller/pull/5982) [#6006](https://github.com/Kong/kubernetes-ingress-controller/pull/6006) + [#6055](https://github.com/Kong/kubernetes-ingress-controller/pull/6055) - Added `FallbackConfiguration` feature gate to enable the controller to generate a fallback configuration for Kong when it fails to apply the original one. The feature gate is disabled by default. [#5993](https://github.com/Kong/kubernetes-ingress-controller/pull/5993) diff --git a/FEATURE_GATES.md b/FEATURE_GATES.md index ae7c071ea8..ce8c070eab 100644 --- a/FEATURE_GATES.md +++ b/FEATURE_GATES.md @@ -71,6 +71,7 @@ Features that reach GA and over time become stable will be removed from this tab | KongServiceFacade | `false` | Alpha | 3.1.0 | TBD | | SanitizeKonnectConfigDumps | `true` | Beta | 3.1.0 | TBD | | FallbackConfiguration | `false` | Alpha | 3.2.0 | TBD | +| KongCustomEntity | `false` | Alpha | 3.2.0 | TBD | **NOTE**: The `Gateway` feature gate refers to [Gateway API](https://github.com/kubernetes-sigs/gateway-api) APIs which are in diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 2566adab5b..2f0eddb391 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -15,6 +15,7 @@ resources: - bases/configuration.konghq.com_kongupstreampolicies.yaml - bases/configuration.konghq.com_kongvaults.yaml - bases/configuration.konghq.com_konglicenses.yaml +- bases/configuration.konghq.com_kongcustomentities.yaml #+kubebuilder:scaffold:crdkustomizeresource # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b200918940..817f3a53a9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -105,6 +105,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/docs/cli-arguments.md b/docs/cli-arguments.md index 73a29e0fcf..a0ad16120d 100644 --- a/docs/cli-arguments.md +++ b/docs/cli-arguments.md @@ -26,6 +26,7 @@ | `--enable-controller-ingress-class-networkingv1` | `bool` | Enable the networking.k8s.io/v1 IngressClass controller. | `true` | | `--enable-controller-ingress-class-parameters` | `bool` | Enable the IngressClassParameters controller. | `true` | | `--enable-controller-ingress-networkingv1` | `bool` | Enable the networking.k8s.io/v1 Ingress controller. | `true` | +| `--enable-controller-kong-custom-entity` | `bool` | Enable the KongCustomEntity controller. | `true` | | `--enable-controller-kong-license` | `bool` | Enable the KongLicense controller. | `true` | | `--enable-controller-kong-service-facade` | `bool` | Enable the KongServiceFacade controller. | `true` | | `--enable-controller-kong-upstream-policy` | `bool` | Enable the KongUpstreamPolicy controller. | `true` | diff --git a/hack/generators/cache-stores/spec.go b/hack/generators/cache-stores/spec.go index befcbbf53d..aadc25f960 100644 --- a/hack/generators/cache-stores/spec.go +++ b/hack/generators/cache-stores/spec.go @@ -108,4 +108,8 @@ var supportedTypes = []cacheStoreSupportedType{ Package: "kongv1alpha1", KeyFunc: clusterWideKeyFunc, }, + { + Type: "KongCustomEntity", + Package: "kongv1alpha1", + }, } diff --git a/hack/generators/controllers/networking/main.go b/hack/generators/controllers/networking/main.go index ee7bc6ab84..9676166637 100644 --- a/hack/generators/controllers/networking/main.go +++ b/hack/generators/controllers/networking/main.go @@ -264,6 +264,23 @@ var inputControllersNeeded = &typesNeeded{ AcceptsIngressClassNameAnnotation: true, RBACVerbs: []string{"get", "list", "watch"}, }, + typeNeeded{ + Group: "configuration.konghq.com", + Version: "v1alpha1", + Kind: "KongCustomEntity", + PackageImportAlias: "kongv1alpha1", + PackageAlias: "KongV1Alpha1", + Package: kongv1alpha1, + Plural: "kongcustomentities", + CacheType: "KongCustomEntity", + NeedsStatusPermissions: true, + ConfigStatusNotificationsEnabled: true, + ProgrammedCondition: ProgrammedConditionConfiguration{ + UpdatesEnabled: true, + }, + AcceptsIngressClassNameAnnotation: true, + RBACVerbs: []string{"get", "list", "watch"}, + }, } var inputRBACPermissionsNeeded = &rbacsNeeded{ diff --git a/internal/controllers/configuration/zz_generated_controllers.go b/internal/controllers/configuration/zz_generated_controllers.go index 0ddb99ca99..16398f0135 100644 --- a/internal/controllers/configuration/zz_generated_controllers.go +++ b/internal/controllers/configuration/zz_generated_controllers.go @@ -2016,6 +2016,171 @@ func (r *KongV1Alpha1KongVaultReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, nil } +// ----------------------------------------------------------------------------- +// KongV1Alpha1 KongCustomEntity - Reconciler +// ----------------------------------------------------------------------------- + +// KongV1Alpha1KongCustomEntityReconciler reconciles KongCustomEntity resources +type KongV1Alpha1KongCustomEntityReconciler struct { + client.Client + + Log logr.Logger + Scheme *runtime.Scheme + DataplaneClient controllers.DataPlane + CacheSyncTimeout time.Duration + StatusQueue *status.Queue + + IngressClassName string + DisableIngressClassLookups bool +} + +var _ controllers.Reconciler = &KongV1Alpha1KongCustomEntityReconciler{} + +// SetupWithManager sets up the controller with the Manager. +func (r *KongV1Alpha1KongCustomEntityReconciler) SetupWithManager(mgr ctrl.Manager) error { + blder := ctrl.NewControllerManagedBy(mgr). + // set the controller name + Named("KongV1Alpha1KongCustomEntity"). + WithOptions(controller.Options{ + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return r.Log + }, + CacheSyncTimeout: r.CacheSyncTimeout, + }) + // if configured, start the status updater controller + if r.StatusQueue != nil { + blder.WatchesRawSource( + source.Channel( + r.StatusQueue.Subscribe(schema.GroupVersionKind{ + Group: "configuration.konghq.com", + Version: "v1alpha1", + Kind: "KongCustomEntity", + }), + &handler.EnqueueRequestForObject{}, + ), + ) + } + if !r.DisableIngressClassLookups { + blder.Watches(&netv1.IngressClass{}, + handler.EnqueueRequestsFromMapFunc(r.listClassless), + builder.WithPredicates(predicate.NewPredicateFuncs(ctrlutils.IsDefaultIngressClass)), + ) + } + preds := ctrlutils.GeneratePredicateFuncsForIngressClassFilter(r.IngressClassName) + return blder.Watches(&kongv1alpha1.KongCustomEntity{}, + &handler.EnqueueRequestForObject{}, + builder.WithPredicates(preds), + ). + Complete(r) +} + +// listClassless finds and reconciles all objects without ingress class information +func (r *KongV1Alpha1KongCustomEntityReconciler) listClassless(ctx context.Context, obj client.Object) []reconcile.Request { + resourceList := &kongv1alpha1.KongCustomEntityList{} + if err := r.Client.List(ctx, resourceList); err != nil { + r.Log.Error(err, "Failed to list classless kongcustomentities") + return nil + } + var recs []reconcile.Request + for i, resource := range resourceList.Items { + if ctrlutils.IsIngressClassEmpty(&resourceList.Items[i]) { + recs = append(recs, reconcile.Request{ + NamespacedName: k8stypes.NamespacedName{ + Namespace: resource.Namespace, + Name: resource.Name, + }, + }) + } + } + return recs +} + +// SetLogger sets the logger. +func (r *KongV1Alpha1KongCustomEntityReconciler) SetLogger(l logr.Logger) { + r.Log = l +} + +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongcustomentities,verbs=get;list;watch +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongcustomentities/status,verbs=get;update;patch + +// Reconcile processes the watched objects +func (r *KongV1Alpha1KongCustomEntityReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("KongV1Alpha1KongCustomEntity", req.NamespacedName) + + // get the relevant object + obj := new(kongv1alpha1.KongCustomEntity) + + if err := r.Get(ctx, req.NamespacedName, obj); err != nil { + if apierrors.IsNotFound(err) { + obj.Namespace = req.Namespace + obj.Name = req.Name + + return ctrl.Result{}, r.DataplaneClient.DeleteObject(obj) + } + return ctrl.Result{}, err + } + log.V(util.DebugLevel).Info("Reconciling resource", "namespace", req.Namespace, "name", req.Name) + + // clean the object up if it's being deleted + if !obj.DeletionTimestamp.IsZero() && time.Now().After(obj.DeletionTimestamp.Time) { + log.V(util.DebugLevel).Info("Resource is being deleted, its configuration will be removed", "type", "KongCustomEntity", "namespace", req.Namespace, "name", req.Name) + + objectExistsInCache, err := r.DataplaneClient.ObjectExists(obj) + if err != nil { + return ctrl.Result{}, err + } + if objectExistsInCache { + if err := r.DataplaneClient.DeleteObject(obj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil // wait until the object is no longer present in the cache + } + return ctrl.Result{}, nil + } + + class := new(netv1.IngressClass) + if !r.DisableIngressClassLookups { + if err := r.Get(ctx, k8stypes.NamespacedName{Name: r.IngressClassName}, class); err != nil { + // we log this without taking action to support legacy configurations that only set ingressClassName or + // used the class annotation and did not create a corresponding IngressClass. We only need this to determine + // if the IngressClass is default or to configure default settings, and can assume no/no additional defaults + // if none exists. + log.V(util.DebugLevel).Info("Could not retrieve IngressClass", "ingressclass", r.IngressClassName) + } + } + // if the object is not configured with our ingress.class, then we need to ensure it's removed from the cache + if !ctrlutils.MatchesIngressClass(obj, r.IngressClassName, ctrlutils.IsDefaultIngressClass(class)) { + log.V(util.DebugLevel).Info("Object missing ingress class, ensuring it's removed from configuration", + "namespace", req.Namespace, "name", req.Name, "class", r.IngressClassName) + return ctrl.Result{}, r.DataplaneClient.DeleteObject(obj) + } else { + log.V(util.DebugLevel).Info("Object has matching ingress class", "namespace", req.Namespace, "name", req.Name, + "class", r.IngressClassName) + } + + // update the kong Admin API with the changes + if err := r.DataplaneClient.UpdateObject(obj); err != nil { + return ctrl.Result{}, err + } + // if status updates are enabled report the status for the object + if r.DataplaneClient.AreKubernetesObjectReportsEnabled() { + log.V(util.DebugLevel).Info("Updating programmed condition status", "namespace", req.Namespace, "name", req.Name) + configurationStatus := r.DataplaneClient.KubernetesObjectConfigurationStatus(obj) + conditions, updateNeeded := ctrlutils.EnsureProgrammedCondition( + configurationStatus, + obj.Generation, + obj.Status.Conditions, + ) + obj.Status.Conditions = conditions + if updateNeeded { + return ctrl.Result{}, r.Status().Update(ctx, obj) + } + log.V(util.DebugLevel).Info("Status update not needed", "namespace", req.Namespace, "name", req.Name) + } + + return ctrl.Result{}, nil +} + // ----------------------------------------------------------------------------- // API Group "" resource nodes // ----------------------------------------------------------------------------- diff --git a/internal/controllers/utils/utils.go b/internal/controllers/utils/utils.go index 1eeea14b93..89b6db4f0d 100644 --- a/internal/controllers/utils/utils.go +++ b/internal/controllers/utils/utils.go @@ -9,6 +9,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/kong/kubernetes-ingress-controller/v3/internal/annotations" + kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" ) const defaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class" @@ -32,6 +33,12 @@ func MatchesIngressClass(obj client.Object, controllerIngressClass string, isDef return true } } + // For KongCustomEntities, we check whether the `spec.ControllerName` matches. + if customEntity, isKongCustomEntity := obj.(*kongv1alpha1.KongCustomEntity); isKongCustomEntity { + if customEntity.Spec.ControllerName == controllerIngressClass { + return true + } + } return objectIngressClass == controllerIngressClass } diff --git a/internal/dataplane/configfetcher/kongrawstate_test.go b/internal/dataplane/configfetcher/kongrawstate_test.go index 338a162b2d..efed5783bd 100644 --- a/internal/dataplane/configfetcher/kongrawstate_test.go +++ b/internal/dataplane/configfetcher/kongrawstate_test.go @@ -344,6 +344,9 @@ func ensureAllKongStateFieldsAreTested(t *testing.T, testedFields []string) { "Plugins", // Licenses are injected from the license getter rather than extracted from the last state. "Licenses", + // CustomEntities are not supported yet because go-database-reconciler does not include custom entities. + // TODO: support custom entities: https://github.com/Kong/kubernetes-ingress-controller/issues/6054 + "CustomEntities", } allKongStateFields := func() []string { var fields []string diff --git a/internal/dataplane/deckgen/deckgen.go b/internal/dataplane/deckgen/deckgen.go index 9f41eb7334..87bd213f12 100644 --- a/internal/dataplane/deckgen/deckgen.go +++ b/internal/dataplane/deckgen/deckgen.go @@ -8,15 +8,24 @@ import ( "github.com/google/go-cmp/cmp" "github.com/kong/go-database-reconciler/pkg/file" "github.com/kong/go-kong/kong" + "github.com/kong/go-kong/kong/custom" ) // GenerateSHA generates a SHA256 checksum of targetContent, with the purpose // of change detection. -func GenerateSHA(targetContent *file.Content) ([]byte, error) { +func GenerateSHA(targetContent *file.Content, customEntities map[string][]custom.Object) ([]byte, error) { jsonConfig, err := gojson.Marshal(targetContent) if err != nil { return nil, fmt.Errorf("marshaling Kong declarative configuration to JSON: %w", err) } + // Calculate SHA including the custom entities. + if len(customEntities) > 0 { + jsonCustomEntities, err := gojson.Marshal(customEntities) + if err != nil { + return nil, fmt.Errorf("marshaling Kong custom entities to JSON: %w", err) + } + jsonConfig = append(jsonConfig, jsonCustomEntities...) + } shaSum := sha256.Sum256(jsonConfig) return shaSum[:], nil diff --git a/internal/dataplane/fallback/graph_dependencies.go b/internal/dataplane/fallback/graph_dependencies.go index 77e285fcf7..dfb3cfc9c7 100644 --- a/internal/dataplane/fallback/graph_dependencies.go +++ b/internal/dataplane/fallback/graph_dependencies.go @@ -63,6 +63,10 @@ func ResolveDependencies(cache store.CacheStores, obj client.Object) ([]client.O *kongv1alpha1.IngressClassParameters, *kongv1alpha1.KongVault: return nil, nil + case *kongv1alpha1.KongCustomEntity: + // TODO: KongCustomEnity is not supported in failure domain yet. + // https://github.com/Kong/kubernetes-ingress-controller/issues/6122 + return nil, nil default: return nil, fmt.Errorf("unsupported object type: %T", obj) } diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go index 3ee98b4484..1fcdb1ca56 100644 --- a/internal/dataplane/kong_client.go +++ b/internal/dataplane/kong_client.go @@ -777,6 +777,13 @@ func (c *KongClient) sendToClient( AppendStubEntityWhenConfigEmpty: !client.IsKonnect() && config.InMemory, } targetContent := deckgen.ToDeckContent(ctx, logger, s, deckGenParams) + customEntities := make(sendconfig.CustomEntitiesByType) + for entityType, collection := range s.CustomEntities { + for _, entity := range collection.Entities { + customEntities[entityType] = append(customEntities[entityType], entity.Object) + } + } + sendDiagnostic := prepareSendDiagnosticFn(ctx, logger, c.diagnostic, s, targetContent, deckGenParams, c.brokenObjects) // apply the configuration update in Kong @@ -788,6 +795,7 @@ func (c *KongClient) sendToClient( client, config, targetContent, + customEntities, c.prometheusMetrics, c.updateStrategyResolver, c.configChangeDetector, diff --git a/internal/dataplane/kongstate/customentity.go b/internal/dataplane/kongstate/customentity.go index de9fadf1d6..9ded140971 100644 --- a/internal/dataplane/kongstate/customentity.go +++ b/internal/dataplane/kongstate/customentity.go @@ -1,6 +1,14 @@ package kongstate -import "github.com/kong/go-kong/kong" +import ( + "context" + "sort" + + "github.com/kong/go-kong/kong" + "github.com/kong/go-kong/kong/custom" + + kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" +) // EntityFieldType represents type of a Kong entity field. // possible field types include boolean, integer, number, string, array, set, map, record, json, foreign. @@ -89,3 +97,63 @@ func ExtractEntityFieldDefinitions(schema kong.Schema) EntitySchema { } return retSchema } + +// IsKnownEntityType returns true if the entities of the type are "standard" and processed elsewhere in KIC. +func IsKnownEntityType(entityType string) bool { + switch entityType { + case + // Types of standard Kong entities that are processed elsewhere in KIC. + // So the entities cannot be specified via KongCustomEntity types. + string(kong.EntityTypeServices), + string(kong.EntityTypeRoutes), + string(kong.EntityTypeUpstreams), + string(kong.EntityTypeTargets), + string(kong.EntityTypeConsumers), + string(kong.EntityTypeConsumerGroups), + string(kong.EntityTypePlugins): + return true + default: + return false + } +} + +// KongCustomEntityCollection is a collection of custom Kong entities with the same type. +type KongCustomEntityCollection struct { + // Schema is the Schema of the entity. + Schema EntitySchema `json:"-"` + // Entities is the list of entities in the collection. + Entities []CustomEntity +} + +// CustomEntity saves content of a Kong custom entity with the pointer to the k8s resource translating to it. +type CustomEntity struct { + custom.Object + // K8sKongCustomEntity refers to the KongCustomEntity resource that translate to it. + K8sKongCustomEntity *kongv1alpha1.KongCustomEntity +} + +// SchemaGetter is the interface to fetch the schema of a Kong entity by its type. +// Used for fetching schema of custom entity for filling "foreign" field referring to other entities. +type SchemaGetter interface { + Get(ctx context.Context, entityType string) (kong.Schema, error) +} + +// sortCustomEntities sorts the custom entities of each type. +// Since there may not be a consistent field to identify an entity, here we sort them by the k8s namespace/name. +func (ks *KongState) sortCustomEntities() { + for _, collection := range ks.CustomEntities { + sort.Slice(collection.Entities, func(i, j int) bool { + e1 := collection.Entities[i] + e2 := collection.Entities[j] + // Compare namespace first. + if e1.K8sKongCustomEntity.Namespace < e2.K8sKongCustomEntity.Namespace { + return true + } + if e1.K8sKongCustomEntity.Namespace > e2.K8sKongCustomEntity.Namespace { + return false + } + // If namespace are the same, compare name. + return e1.K8sKongCustomEntity.Name < e2.K8sKongCustomEntity.Name + }) + } +} diff --git a/internal/dataplane/kongstate/customentity_test.go b/internal/dataplane/kongstate/customentity_test.go index ff754de6df..64ff4d049e 100644 --- a/internal/dataplane/kongstate/customentity_test.go +++ b/internal/dataplane/kongstate/customentity_test.go @@ -4,7 +4,11 @@ import ( "testing" "github.com/kong/go-kong/kong" + "github.com/kong/go-kong/kong/custom" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1" ) func TestExtractEntityFieldDefinitions(t *testing.T) { @@ -98,3 +102,109 @@ func TestExtractEntityFieldDefinitions(t *testing.T) { }) } } + +func TestSortCustomEntities(t *testing.T) { + tesCases := []struct { + name string + customEntityCollections map[string]*KongCustomEntityCollection + sortedCollections map[string]*KongCustomEntityCollection + }{ + { + name: "custom entities in multiple namespaces", + customEntityCollections: map[string]*KongCustomEntityCollection{ + "foo": { + Entities: []CustomEntity{ + { + Object: custom.Object{ + "name": "e1", + "key": "value1", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aab", + Namespace: "bbb", + }, + }, + }, + { + Object: custom.Object{ + "name": "e2", + "key": "value2", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abc", + Namespace: "bbb", + }, + }, + }, + { + Object: custom.Object{ + "name": "e3", + "key": "value3", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abc", + Namespace: "aaa", + }, + }, + }, + }, + }, + }, + sortedCollections: map[string]*KongCustomEntityCollection{ + "foo": { + Entities: []CustomEntity{ + { + Object: custom.Object{ + "name": "e3", + "key": "value3", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abc", + Namespace: "aaa", + }, + }, + }, + { + Object: custom.Object{ + "name": "e1", + "key": "value1", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aab", + Namespace: "bbb", + }, + }, + }, + { + Object: custom.Object{ + "name": "e2", + "key": "value2", + }, + K8sKongCustomEntity: &kongv1alpha1.KongCustomEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abc", + Namespace: "bbb", + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range tesCases { + t.Run(tc.name, func(t *testing.T) { + ks := &KongState{ + CustomEntities: tc.customEntityCollections, + } + ks.sortCustomEntities() + require.Equal(t, tc.sortedCollections, ks.CustomEntities) + }) + } +} diff --git a/internal/dataplane/kongstate/kongstate.go b/internal/dataplane/kongstate/kongstate.go index 75169adce8..b9c3bbb0be 100644 --- a/internal/dataplane/kongstate/kongstate.go +++ b/internal/dataplane/kongstate/kongstate.go @@ -1,7 +1,9 @@ package kongstate import ( + "context" "crypto/sha256" + "encoding/json" "fmt" "strconv" "strings" @@ -37,6 +39,8 @@ type KongState struct { Consumers []Consumer ConsumerGroups []ConsumerGroup Vaults []Vault + + CustomEntities map[string]*KongCustomEntityCollection } // SanitizedCopy returns a shallow copy with sensitive values redacted best-effort. @@ -68,6 +72,7 @@ func (ks *KongState) SanitizedCopy(uuidGenerator util.UUIDGenerator) *KongState }(), ConsumerGroups: ks.ConsumerGroups, Vaults: ks.Vaults, + CustomEntities: ks.CustomEntities, } } @@ -361,22 +366,10 @@ func (ks *KongState) getPluginRelations(cacheStore store.Storer, log logr.Logger // // Code in buildPlugins() will combine plugin associations into // multi-entity plugins within the local namespace - namespace := referer.GetNamespace() - if plugin.Namespace != "" { - // remote KongPlugin, permitted if ReferenceGrant allows - if err := isRemotePluginReferenceAllowed( - cacheStore, - pluginReference{ - Referer: referer, - Namespace: plugin.Namespace, - Name: plugin.Name, - }, - ); err != nil { - log.Error(err, "could not bind requested plugin", "plugin", plugin.Name, "namespace", plugin.Namespace) - return - } - - namespace = plugin.Namespace + namespace, err := extractReferredPluginNamespace(cacheStore, referer, plugin) + if err != nil { + log.Error(err, "could not bind requested plugin", "plugin", plugin.Name, "namespace", plugin.Namespace) + return } pluginKey := namespace + ":" + plugin.Name @@ -690,3 +683,305 @@ func maybeLogKongIngressDeprecationError(logger logr.Logger, services []*corev1. } } } + +// FillCustomEntities fills custom entities in KongState. +func (ks *KongState) FillCustomEntities( + logger logr.Logger, + s store.Storer, + failuresCollector *failures.ResourceFailuresCollector, + schemaGetter SchemaGetter, + workspace string, +) { + entities := s.ListKongCustomEntities() + if len(entities) == 0 { + return + } + logger = logger.WithName("fillCustomEntities") + + if ks.CustomEntities == nil { + ks.CustomEntities = map[string]*KongCustomEntityCollection{} + } + // Fetch relations between plugins and services/routes/consumers and store the pointer to translated Kong entities. + // Used for fetching entity referred by a custom entity and fill the ID of referred entity. + pluginRels := ks.getPluginRelatedEntitiesRef(s, logger) + + for _, entity := range entities { + // reject the custom entity if its type is in "known" entity types that are already processed. + if IsKnownEntityType(entity.Spec.EntityType) { + failuresCollector.PushResourceFailure( + fmt.Sprintf("cannot use known entity type %s in custom entity", entity.Spec.EntityType), + entity, + ) + continue + } + // Fetch the entity schema. + schema, err := ks.fetchEntitySchema(schemaGetter, entity.Spec.EntityType) + if err != nil { + failuresCollector.PushResourceFailure( + fmt.Sprintf("failed to fetch entity schema for entity type %s: %v", entity.Spec.EntityType, err), + entity, + ) + continue + } + // Unmarshal fields of the entity. + var parsedEntity map[string]interface{} + if err = json.Unmarshal(entity.Spec.Fields.Raw, &parsedEntity); err != nil { + failuresCollector.PushResourceFailure(fmt.Sprintf("failed to unmarshal fields of entity: %v", err), entity) + continue + } + // Fill the "foreign" fields if the entity has such fields referencing services/routes/consumers. + ks.fillCustomEntityForeignFields(logger, entity, schema, parsedEntity, pluginRels, workspace) + // Put the entity into the custom collection to store the entities of its type. + if _, ok := ks.CustomEntities[entity.Spec.EntityType]; !ok { + ks.CustomEntities[entity.Spec.EntityType] = &KongCustomEntityCollection{ + Schema: schema, + } + } + collection := ks.CustomEntities[entity.Spec.EntityType] + collection.Entities = append(collection.Entities, CustomEntity{ + Object: parsedEntity, + K8sKongCustomEntity: entity, + }) + } + + ks.sortCustomEntities() +} + +// fetchEntitySchema fetches schema of an entity by its type and stores the schema in its custom entity collection +// as a cache to avoid excessive calling of Kong admin APIs. +func (ks *KongState) fetchEntitySchema(schemaGetter SchemaGetter, entityType string) (EntitySchema, error) { + collection, ok := ks.CustomEntities[entityType] + if ok { + return collection.Schema, nil + } + // Use `context.Background()` here because `BuildKongConfig` does not provide a context. + schema, err := schemaGetter.Get(context.Background(), entityType) + if err != nil { + return EntitySchema{}, err + } + return ExtractEntityFieldDefinitions(schema), nil +} + +// fillCustomEntityForeignFields fills the "foreign" fields of a custom Kong entity +// if it refers to a service/route/consumer. +// Because Kong gateway requires the "foreign" fields to use IDs of the referred entity, +// it fills ID of the referred entity and fill the ID of the entity to the field. +// So the function has the side effect that the referred entity will have a generated fixed ID. +// It does not support fields referring to other types of entities. +func (ks *KongState) fillCustomEntityForeignFields( + logger logr.Logger, + k8sEntity *kongv1alpha1.KongCustomEntity, + schema EntitySchema, + parsedEntity map[string]any, + pluginRelEntities PluginRelatedEntitiesRefs, + workspace string, +) { + logger = logger.WithValues("entity_namespace", k8sEntity.Namespace, "entity_name", k8sEntity.Name) + // Find referred entity via the plugin in its spec.parentRef. + // Then we can fetch the referred service/route/consumer from the reference relations of the plugin. + parentRef := k8sEntity.Spec.ParentRef + var namespace string + // Abort if the parentRef is empty or does not refer to a plugin. + if parentRef == nil || + (parentRef.Group == nil || *parentRef.Group != kongv1alpha1.GroupVersion.Group) { + return + } + if parentRef.Kind == nil || (*parentRef.Kind != "KongPlugin" && *parentRef.Kind != "KongClusterPlugin") { + return + } + // Extract the plugin key to get the plugin relations. + if parentRef.Namespace == nil || *parentRef.Namespace == "" { + namespace = k8sEntity.Namespace + } else { + namespace = *parentRef.Namespace + } + pluginKey := namespace + ":" + parentRef.Name + // Get the relations with other entities of the plugin. + rels, ok := pluginRelEntities.RelatedEntities[pluginKey] + if !ok { + return + } + logger.V(util.DebugLevel).Info("fetch references via plugin", "plugin_key", pluginKey) + + // Traverse through the fields of the entity and fill the "foreign" fields with IDs of referring entities. + // Note: this procedure will make referred services'/routes'/consumers' ID to be filled. + // So it requires the `FillIDs` feature gate to be enabled. + for fieldName, field := range schema.Fields { + if field.Type != EntityFieldTypeForeign { + continue + } + switch field.Reference { + case string(kong.EntityTypeServices): + serviceIDs := getServiceIDFromPluginRels(logger, rels, pluginRelEntities.RouteAttachedService, workspace) + // TODO: we should generate multiple entities if the plugin is attached to multiple services/routes/consumers. + // https://github.com/Kong/kubernetes-ingress-controller/issues/6123 + if len(serviceIDs) > 0 { + parsedEntity[fieldName] = map[string]interface{}{ + "id": serviceIDs[0], + } + logger.V(util.DebugLevel).Info("added ref to service", "service_id", serviceIDs[0]) + } + case string(kong.EntityTypeRoutes): + routeIDs := lo.FilterMap(rels.Routes, func(r *Route, _ int) (string, bool) { + if err := r.FillID(workspace); err != nil { + return "", false + } + return *r.ID, true + }) + if len(routeIDs) > 0 { + parsedEntity[fieldName] = map[string]interface{}{ + "id": routeIDs[0], + } + logger.V(util.DebugLevel).Info("added ref to route", "route_id", routeIDs[0]) + } + case string(kong.EntityTypeConsumers): + consumerIDs := lo.FilterMap(rels.Consumers, func(c *Consumer, _ int) (string, bool) { + if err := c.FillID(workspace); err != nil { + return "", false + } + return *c.ID, true + }) + if len(consumerIDs) > 0 { + parsedEntity[fieldName] = map[string]interface{}{ + "id": consumerIDs[0], + } + logger.V(util.DebugLevel).Info("added ref to consumer", "consumer_id", consumerIDs[0]) + } + } + } +} + +// getServiceIDFromPluginRels returns the ID of the services which a plugin refers to in RelatedEntitiesRef. +// It fills the IDs of services directly referred, and IDs of services where referred routes attaches to. +func getServiceIDFromPluginRels(log logr.Logger, rels RelatedEntitiesRef, routeAttachedService map[string]*Service, workspace string) []string { + // Return IDs of directly referred services. + if len(rels.Services) > 0 { + return lo.FilterMap(rels.Services, func(s *Service, _ int) (string, bool) { + if err := s.FillID(workspace); err != nil { + log.Error(err, "failed to fill ID for service") + return "", false + } + return *s.ID, true + }, + ) + } + // Returns IDs of services where the referred routes attaches. + if len(rels.Routes) > 0 { + serviceIDs := lo.FilterMap( + rels.Routes, func(r *Route, _ int) (string, bool) { + svc, ok := routeAttachedService[*r.Name] + if !ok { + return "", false + } + if err := svc.FillID(workspace); err != nil { + log.Error(err, "failed to fill ID for service") + return "", false + } + return *svc.ID, true + }, + ) + return lo.Uniq(serviceIDs) + } + return nil +} + +// getPluginRelatedEntitiesRef gets services/routes/consumers referred by each plugin and returns the pointer to them. +// It is for the custom entities to fill the IDs of the referred entities into their "foreign" fields. +// It basically does the same thing as getPluginRelations but stores the pointers +// because we need to call the FillID method of the entities to fetch the ID, +// as Kong gateway requires IDs in the "foreign" fields (not other identifiers such as name.) +// +// TODO: refactor the building of plugin related entities and share the result between here and building plugins: +// https://github.com/Kong/kubernetes-ingress-controller/issues/6115 +func (ks *KongState) getPluginRelatedEntitiesRef(cacheStore store.Storer, log logr.Logger) PluginRelatedEntitiesRefs { + pluginRels := PluginRelatedEntitiesRefs{ + RelatedEntities: map[string]RelatedEntitiesRef{}, + RouteAttachedService: map[string]*Service{}, + } + addRelation := func(referer client.Object, plugin annotations.NamespacedKongPlugin, entity any) { + namespace, err := extractReferredPluginNamespace(cacheStore, referer, plugin) + if err != nil { + log.Error(err, "could not bind requested plugin", "plugin", plugin.Name, "namespace", plugin.Namespace) + return + } + pluginKey := namespace + ":" + plugin.Name + relations, ok := pluginRels.RelatedEntities[pluginKey] + if !ok { + relations = RelatedEntitiesRef{} + } + switch e := entity.(type) { + case *Consumer: + relations.Consumers = append(relations.Consumers, e) + case *Route: + relations.Routes = append(relations.Routes, e) + case *Service: + relations.Services = append(relations.Services, e) + } + pluginRels.RelatedEntities[pluginKey] = relations + } + + for i := range ks.Services { + for _, svc := range ks.Services[i].K8sServices { + pluginList := annotations.ExtractNamespacedKongPluginsFromAnnotations(svc.GetAnnotations()) + for _, plugin := range pluginList { + addRelation(svc, plugin, &ks.Services[i]) + } + } + + for j, r := range ks.Services[i].Routes { + ingress := ks.Services[i].Routes[j].Ingress + pluginList := annotations.ExtractNamespacedKongPluginsFromAnnotations(ingress.Annotations) + for _, plugin := range pluginList { + // Pretend we have a full Ingress struct for reference checks. + virtualIngress := netv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ingress.Namespace, + Name: ingress.Name, + }, + } + addRelation(&virtualIngress, plugin, &ks.Services[i].Routes[j]) + // For some entities, we need to find the referred service via route + // but the `Service` field of translated routes are empty because we do not want the field to appear in final declarative config. + // So we need to maintain a map from route name to service to find the service object. + pluginRels.RouteAttachedService[*r.Name] = &ks.Services[i] + } + } + } + + for i, c := range ks.Consumers { + pluginList := annotations.ExtractNamespacedKongPluginsFromAnnotations(c.K8sKongConsumer.GetAnnotations()) + for _, plugin := range pluginList { + addRelation(&c.K8sKongConsumer, plugin, &ks.Consumers[i]) + } + } + return pluginRels +} + +func extractReferredPluginNamespace( + cacheStore store.Storer, referer client.Object, plugin annotations.NamespacedKongPlugin, +) (string, error) { + // There are 2 types of KongPlugin references: local and remote. + // A local reference is one where the KongPlugin is in the same namespace as the referer. + // A remote reference is one where the KongPlugin is in a different namespace. + // By default a KongPlugin is considered local. + // If the plugin has a namespace specified, it is considered remote. + // + // The referer is the entity that the KongPlugin is associated with. + if plugin.Namespace == "" { + return referer.GetNamespace(), nil + } + + // remote KongPlugin, permitted if ReferenceGrant allows. + err := isRemotePluginReferenceAllowed( + cacheStore, + pluginReference{ + Referer: referer, + Namespace: plugin.Namespace, + Name: plugin.Name, + }, + ) + if err != nil { + return "", err + } + return plugin.Namespace, nil +} diff --git a/internal/dataplane/kongstate/kongstate_test.go b/internal/dataplane/kongstate/kongstate_test.go index b9bbcf21bc..83f42514e4 100644 --- a/internal/dataplane/kongstate/kongstate_test.go +++ b/internal/dataplane/kongstate/kongstate_test.go @@ -1,6 +1,7 @@ package kongstate import ( + "context" "fmt" "reflect" "strconv" @@ -12,6 +13,7 @@ import ( "github.com/go-logr/logr/testr" "github.com/go-logr/zapr" "github.com/kong/go-kong/kong" + "github.com/kong/go-kong/kong/custom" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -76,6 +78,25 @@ func TestKongState_SanitizedCopy(t *testing.T) { }, }, }, + CustomEntities: map[string]*KongCustomEntityCollection{ + "test_entities": { + Schema: EntitySchema{ + Fields: map[string]EntityField{ + "name": { + Type: EntityFieldTypeString, + Required: true, + }, + }, + }, + Entities: []CustomEntity{ + { + Object: map[string]interface{}{ + "name": "foo", + }, + }, + }, + }, + }, }, want: KongState{ Services: []Service{{Service: kong.Service{ID: kong.String("1")}}}, @@ -100,6 +121,25 @@ func TestKongState_SanitizedCopy(t *testing.T) { }, }, }, + CustomEntities: map[string]*KongCustomEntityCollection{ + "test_entities": { + Schema: EntitySchema{ + Fields: map[string]EntityField{ + "name": { + Type: EntityFieldTypeString, + Required: true, + }, + }, + }, + Entities: []CustomEntity{ + { + Object: map[string]interface{}{ + "name": "foo", + }, + }, + }, + }, + }, }, }, } { @@ -450,9 +490,13 @@ func TestGetPluginRelations(t *testing.T) { "ns1:foo": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}}, "ns1:bar": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Service: []string{"foo-service"}}, "ns1:foobar": {Consumer: []string{"bar-consumer"}}, - "ns2:foo": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Route: []string{"foo-route"}}, - "ns2:bar": {Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group", "bar-consumer-group"}, Route: []string{"foo-route", "bar-route"}}, - "ns2:baz": {Route: []string{"bar-route"}, ConsumerGroup: []string{"bar-consumer-group"}}, + "ns2:foo": { + Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group"}, Route: []string{"foo-route"}, + }, + "ns2:bar": { + Consumer: []string{"foo-consumer"}, ConsumerGroup: []string{"foo-consumer-group", "bar-consumer-group"}, Route: []string{"foo-route", "bar-route"}, + }, + "ns2:baz": {Route: []string{"bar-route"}, ConsumerGroup: []string{"bar-consumer-group"}}, }, }, } @@ -1475,3 +1519,301 @@ func TestFillOverrides_ServiceFailures(t *testing.T) { }) } } + +type fakeSchemaGetter struct { + schemas map[string]kong.Schema +} + +var _ SchemaGetter = &fakeSchemaGetter{} + +func (s *fakeSchemaGetter) Get(_ context.Context, entityType string) (kong.Schema, error) { + schema, ok := s.schemas[entityType] + if !ok { + return nil, fmt.Errorf("schema not found") + } + return schema, nil +} + +func TestKongState_FillCustomEntities(t *testing.T) { + customEntityTypeMeta := metav1.TypeMeta{ + APIVersion: kongv1alpha1.GroupVersion.Group + "/" + kongv1alpha1.GroupVersion.Version, + Kind: "KongCustomEntity", + } + kongService1 := kong.Service{ + Name: kong.String("service1"), + } + getKongServiceID := func(s *kong.Service) string { + err := s.FillID("") + require.NoError(t, err) + return *s.ID + } + + testCases := []struct { + name string + initialState *KongState + customEntities []*kongv1alpha1.KongCustomEntity + plugins []*kongv1.KongPlugin + schemas map[string]kong.Schema + expectedCustomEntities map[string][]custom.Object + expectedTranslationFailures map[k8stypes.NamespacedName]string + }{ + { + name: "single custom entity", + initialState: &KongState{}, + customEntities: []*kongv1alpha1.KongCustomEntity{ + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "session-foo", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "sessions", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"session1"}`), + }, + }, + }, + }, + schemas: map[string]kong.Schema{ + "sessions": { + "fields": []interface{}{ + map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "required": true, + }, + }, + }, + }, + }, + expectedCustomEntities: map[string][]custom.Object{ + "sessions": { + { + "name": "session1", + }, + }, + }, + }, + { + name: "custom entity with unknown type", + initialState: &KongState{}, + customEntities: []*kongv1alpha1.KongCustomEntity{ + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "session-foo", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "sessions", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"session1"}`), + }, + }, + }, + }, + expectedTranslationFailures: map[k8stypes.NamespacedName]string{ + { + Namespace: "default", + Name: "session-foo", + }: "failed to fetch entity schema for entity type sessions: schema not found", + }, + }, + { + name: "multiple custom entities with same type", + initialState: &KongState{}, + customEntities: []*kongv1alpha1.KongCustomEntity{ + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "session-foo", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "sessions", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"session-foo"}`), + }, + }, + }, + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "session-bar", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "sessions", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"session-bar"}`), + }, + }, + }, + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default-1", + Name: "session-foo", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "sessions", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"name":"session-foo-1"}`), + }, + }, + }, + }, + schemas: map[string]kong.Schema{ + "sessions": { + "fields": []interface{}{ + map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "required": true, + }, + }, + }, + }, + }, + expectedCustomEntities: map[string][]custom.Object{ + // Should be sorted by original KCE namespace/name. + "sessions": { + { + // from default/bar + "name": "session-bar", + }, + { + // from default/foo + "name": "session-foo", + }, + { + // from default-1/foo + "name": "session-foo-1", + }, + }, + }, + }, + { + name: "custom entities with reference to other entities (services)", + initialState: &KongState{ + Services: []Service{ + { + Service: kongService1, + K8sServices: map[string]*corev1.Service{ + "default/service1": { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "service1", + Annotations: map[string]string{ + annotations.AnnotationPrefix + annotations.PluginsKey: "degraphql-1", + }, + }, + }, + }, + }, // Service: service1 + }, // Services + }, + customEntities: []*kongv1alpha1.KongCustomEntity{ + { + TypeMeta: customEntityTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "degraphql-1", + }, + Spec: kongv1alpha1.KongCustomEntitySpec{ + EntityType: "degraphql_routes", + ControllerName: annotations.DefaultIngressClass, + Fields: apiextensionsv1.JSON{ + Raw: []byte(`{"uri":"/api/me"}`), + }, + ParentRef: &kongv1alpha1.ObjectReference{ + Group: kong.String(kongv1.GroupVersion.Group), + Kind: kong.String("KongPlugin"), + Name: "degraphql-1", + }, + }, + }, + }, + plugins: []*kongv1.KongPlugin{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "degraphql-1", + }, + PluginName: "degraphql", + }, + }, + schemas: map[string]kong.Schema{ + "degraphql_routes": { + "fields": []interface{}{ + map[string]interface{}{ + "uri": map[string]interface{}{ + "type": "string", + "required": true, + }, + }, + map[string]interface{}{ + "service": map[string]interface{}{ + "type": "foreign", + "reference": "services", + }, + }, + }, + }, + }, + expectedCustomEntities: map[string][]custom.Object{ + "degraphql_routes": { + { + "uri": "/api/me", + "service": map[string]interface{}{ + // ID generated from Kong service "service1" in workspace "". + "id": getKongServiceID(&kongService1), + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s, err := store.NewFakeStore(store.FakeObjects{ + KongCustomEntities: tc.customEntities, + KongPlugins: tc.plugins, + }) + require.NoError(t, err) + failuresCollector := failures.NewResourceFailuresCollector(logr.Discard()) + + ks := tc.initialState + ks.FillCustomEntities( + logr.Discard(), s, + failuresCollector, + &fakeSchemaGetter{schemas: tc.schemas}, "", + ) + for entityType, expectedObjectList := range tc.expectedCustomEntities { + require.NotNil(t, ks.CustomEntities[entityType]) + objectList := lo.Map(ks.CustomEntities[entityType].Entities, func(e CustomEntity, _ int) custom.Object { + return e.Object + }) + require.Equal(t, expectedObjectList, objectList) + } + + translationFailures := failuresCollector.PopResourceFailures() + for nsName, message := range tc.expectedTranslationFailures { + hasError := lo.ContainsBy(translationFailures, func(f failures.ResourceFailure) bool { + fmt.Println(f.Message()) + return f.Message() == message && lo.ContainsBy(f.CausingObjects(), func(o client.Object) bool { + return o.GetNamespace() == nsName.Namespace && o.GetName() == nsName.Name + }) + }) + require.Truef(t, hasError, "translation error for KongCustomEntity %s not found", nsName) + } + }) + } +} diff --git a/internal/dataplane/kongstate/plugin.go b/internal/dataplane/kongstate/plugin.go index 598b023f51..2305f4931f 100644 --- a/internal/dataplane/kongstate/plugin.go +++ b/internal/dataplane/kongstate/plugin.go @@ -463,3 +463,14 @@ func (p plugin) toKongPlugin() kong.Plugin { } return result } + +type RelatedEntitiesRef struct { + Services []*Service + Routes []*Route + Consumers []*Consumer +} + +type PluginRelatedEntitiesRefs struct { + RelatedEntities map[string]RelatedEntitiesRef + RouteAttachedService map[string]*Service +} diff --git a/internal/dataplane/sendconfig/inmemory.go b/internal/dataplane/sendconfig/inmemory.go index 2da7f8a7ea..21699002f2 100644 --- a/internal/dataplane/sendconfig/inmemory.go +++ b/internal/dataplane/sendconfig/inmemory.go @@ -11,6 +11,7 @@ import ( "github.com/kong/go-database-reconciler/pkg/file" "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" + "github.com/kong/kubernetes-ingress-controller/v3/internal/util" ) type ConfigService interface { @@ -54,6 +55,22 @@ func (s UpdateStrategyInMemory) Update(ctx context.Context, targetState ContentW if err != nil { return fmt.Errorf("constructing kong configuration: %w", err) } + + if len(targetState.CustomEntities) > 0 { + unmarshaledConfig := map[string]any{} + if err := json.Unmarshal(config, &unmarshaledConfig); err != nil { + return fmt.Errorf("unmarshaling config for adding custom entities: %w", err) + } + for entityType, entities := range targetState.CustomEntities { + unmarshaledConfig[entityType] = entities + s.logger.V(util.DebugLevel).Info("Filled custom entities", "entity_type", entityType) + } + config, err = json.Marshal(unmarshaledConfig) + if err != nil { + return fmt.Errorf("constructing kong configuration again with custom entities: %w", err) + } + } + if errBody, reloadConfigErr := s.configService.ReloadDeclarativeRawConfig(ctx, bytes.NewReader(config), true, true); reloadConfigErr != nil { resourceErrors, parseErr := parseFlatEntityErrors(errBody, s.logger) if parseErr != nil { diff --git a/internal/dataplane/sendconfig/sendconfig.go b/internal/dataplane/sendconfig/sendconfig.go index d6fd1174c4..7946ce21b7 100644 --- a/internal/dataplane/sendconfig/sendconfig.go +++ b/internal/dataplane/sendconfig/sendconfig.go @@ -41,13 +41,14 @@ func PerformUpdate( client AdminAPIClient, config Config, targetContent *file.Content, + customEntities CustomEntitiesByType, promMetrics *metrics.CtrlFuncMetrics, updateStrategyResolver UpdateStrategyResolver, configChangeDetector ConfigurationChangeDetector, isFallback bool, ) ([]byte, error) { oldSHA := client.LastConfigSHA() - newSHA, err := deckgen.GenerateSHA(targetContent) + newSHA, err := deckgen.GenerateSHA(targetContent, customEntities) if err != nil { return oldSHA, fmt.Errorf("failed to generate SHA for target content: %w", err) } @@ -72,8 +73,9 @@ func PerformUpdate( logger = logger.WithValues("update_strategy", updateStrategy.Type()) timeStart := time.Now() err = updateStrategy.Update(ctx, ContentWithHash{ - Content: targetContent, - Hash: newSHA, + Content: targetContent, + CustomEntities: customEntities, + Hash: newSHA, }) duration := time.Since(timeStart) diff --git a/internal/dataplane/sendconfig/strategy.go b/internal/dataplane/sendconfig/strategy.go index c81dd5f600..2dc02ea36f 100644 --- a/internal/dataplane/sendconfig/strategy.go +++ b/internal/dataplane/sendconfig/strategy.go @@ -7,15 +7,22 @@ import ( "github.com/kong/go-database-reconciler/pkg/dump" "github.com/kong/go-database-reconciler/pkg/file" "github.com/kong/go-kong/kong" + "github.com/kong/go-kong/kong/custom" "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/metrics" ) +// CustomEntitiesByType stores all custom entities by types. +// The key is the type of the entity, +// and the corresponding slice stores the sorted list of custom entities with that type. +type CustomEntitiesByType map[string][]custom.Object + // ContentWithHash encapsulates file.Content along with its precalculated hash. type ContentWithHash struct { - Content *file.Content - Hash []byte + Content *file.Content + CustomEntities CustomEntitiesByType + Hash []byte } // UpdateStrategy is the way we approach updating data-plane's configuration, depending on its type. diff --git a/internal/dataplane/translator/golden_test.go b/internal/dataplane/translator/golden_test.go index b787f6e289..79977fb16a 100644 --- a/internal/dataplane/translator/golden_test.go +++ b/internal/dataplane/translator/golden_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/go-logr/zapr" + "github.com/kong/go-kong/kong" "github.com/samber/lo" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -49,6 +50,12 @@ const ( settingsFileSuffix = "_settings.yaml" ) +type fakeSchemaServiceProvier struct{} + +func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService { + return translator.UnavailableSchemaService{} +} + // TestTranslator_GoldenTests runs the golden tests for the translator. // // Command to update the golden files: @@ -223,7 +230,7 @@ func runTranslatorGoldenTest(t *testing.T, tc translatorGoldenTestCase) { // Create the translator. s := store.New(cacheStores, "kong", logger) - p, err := translator.NewTranslator(logger, s, "", tc.featureFlags) + p, err := translator.NewTranslator(logger, s, "", tc.featureFlags, fakeSchemaServiceProvier{}) require.NoError(t, err, "failed creating translator") // MustBuild the Kong configuration. diff --git a/internal/dataplane/translator/translator.go b/internal/dataplane/translator/translator.go index 4806ea0d4d..bac5e82f50 100644 --- a/internal/dataplane/translator/translator.go +++ b/internal/dataplane/translator/translator.go @@ -1,7 +1,11 @@ package translator import ( + "context" + "errors" + "github.com/go-logr/logr" + "github.com/kong/go-kong/kong" "sigs.k8s.io/controller-runtime/pkg/client" dpconf "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/config" @@ -45,6 +49,9 @@ type FeatureFlags struct { // KongServiceFacade indicates whether we should support KongServiceFacades as Ingress backends. KongServiceFacade bool + + // KongCustomEntity indicates whether we should support translating custom entities from KongCustomEntity CRs. + KongCustomEntity bool } func NewFeatureFlags( @@ -60,9 +67,15 @@ func NewFeatureFlags( FillIDs: featureGates.Enabled(featuregates.FillIDsFeature), RewriteURIs: featureGates.Enabled(featuregates.RewriteURIsFeature), KongServiceFacade: featureGates.Enabled(featuregates.KongServiceFacade), + KongCustomEntity: featureGates.Enabled(featuregates.KongCustomEntity), } } +// SchemaServiceProvider returns a kong schema service required for translating custom entities. +type SchemaServiceProvider interface { + GetSchemaService() kong.AbstractSchemaService +} + // Translator translates Kubernetes objects and configurations into their // equivalent Kong objects and configurations, producing a complete // state configuration for the Kong Admin API. @@ -73,6 +86,9 @@ type Translator struct { licenseGetter license.Getter featureFlags FeatureFlags + // schemaServiceProvider provides the schema service required for fetching schemas of custom entities. + schemaServiceProvider SchemaServiceProvider + failuresCollector *failures.ResourceFailuresCollector translatedObjectsCollector *ObjectsCollector } @@ -84,6 +100,7 @@ func NewTranslator( storer store.Storer, workspace string, featureFlags FeatureFlags, + schemaServiceProvider SchemaServiceProvider, ) (*Translator, error) { failuresCollector := failures.NewResourceFailuresCollector(logger) @@ -98,6 +115,7 @@ func NewTranslator( storer: storer, workspace: workspace, featureFlags: featureFlags, + schemaServiceProvider: schemaServiceProvider, failuresCollector: failuresCollector, translatedObjectsCollector: translatedObjectsCollector, }, nil @@ -188,6 +206,17 @@ func (t *Translator) BuildKongConfig() KongConfigBuildingResult { t.registerSuccessfullyTranslatedObject(result.Plugins[i].K8sParent) } + // process custom entities + if t.featureFlags.KongCustomEntity { + result.FillCustomEntities(t.logger, t.storer, t.failuresCollector, t.schemaServiceProvider.GetSchemaService(), t.workspace) + // Register successcully translated KCEs to set the status of these KCEs. + for _, collection := range result.CustomEntities { + for i := range collection.Entities { + t.registerSuccessfullyTranslatedObject(collection.Entities[i].K8sKongCustomEntity) + } + } + } + // generate Certificates and SNIs ingressCerts := t.getCerts(ingressRules.SecretNameToSNIs) gatewayCerts := t.getGatewayCerts() @@ -249,3 +278,17 @@ func (t *Translator) registerSuccessfullyTranslatedObject(obj client.Object) { func (t *Translator) popConfiguredKubernetesObjects() []client.Object { return t.translatedObjectsCollector.Pop() } + +// UnavailableSchemaService is a fake schema service used when no gateway admin API clients available. +// It always returns error in its Get and Validate methods. +type UnavailableSchemaService struct{} + +var _ kong.AbstractSchemaService = UnavailableSchemaService{} + +func (s UnavailableSchemaService) Get(_ context.Context, _ string) (kong.Schema, error) { + return nil, errors.New("schema service unavailable") +} + +func (s UnavailableSchemaService) Validate(_ context.Context, _ kong.EntityType, _ any) (bool, string, error) { + return false, "", errors.New("schema service unavailable") +} diff --git a/internal/dataplane/translator/translator_test.go b/internal/dataplane/translator/translator_test.go index 03f6171d36..8fca543c98 100644 --- a/internal/dataplane/translator/translator_test.go +++ b/internal/dataplane/translator/translator_test.go @@ -5019,13 +5019,20 @@ func TestTranslator_ConfiguredKubernetesObjects(t *testing.T) { } } +type fakeSchemaServiceProvier struct{} + +func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService { + return UnavailableSchemaService{} +} + func mustNewTranslator(t *testing.T, storer store.Storer) *Translator { p, err := NewTranslator(zapr.NewLogger(zap.NewNop()), storer, "", FeatureFlags{ // We'll assume these are true for all tests. FillIDs: true, ReportConfiguredKubernetesObjects: true, KongServiceFacade: true, - }) + }, fakeSchemaServiceProvier{}, + ) require.NoError(t, err) return p } diff --git a/internal/manager/config.go b/internal/manager/config.go index aa1df49a73..ab2b853104 100644 --- a/internal/manager/config.go +++ b/internal/manager/config.go @@ -116,6 +116,7 @@ type Config struct { KongServiceFacadeEnabled bool KongVaultEnabled bool KongLicenseEnabled bool + KongCustomEntityEnabled bool // Gateway API toggling. GatewayAPIGatewayController bool @@ -268,6 +269,7 @@ func (c *Config) FlagSet() *pflag.FlagSet { flagSet.BoolVar(&c.KongServiceFacadeEnabled, "enable-controller-kong-service-facade", true, "Enable the KongServiceFacade controller.") flagSet.BoolVar(&c.KongVaultEnabled, "enable-controller-kong-vault", true, "Enable the KongVault controller.") flagSet.BoolVar(&c.KongLicenseEnabled, "enable-controller-kong-license", true, "Enable the KongLicense controller.") + flagSet.BoolVar(&c.KongCustomEntityEnabled, "enable-controller-kong-custom-entity", true, "Enable the KongCustomEntity controller.") // Admission Webhook server config flagSet.StringVar(&c.AdmissionServer.ListenAddr, "admission-webhook-listen", "off", diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 5f5bdd68d1..a5adfd3484 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -296,6 +296,20 @@ func setupControllers( StatusQueue: kubernetesStatusQueue, }, }, + { + Enabled: featureGates.Enabled(featuregates.KongCustomEntity) && c.KongCustomEntityEnabled, + Controller: &configuration.KongV1Alpha1KongCustomEntityReconciler{ + Client: mgr.GetClient(), + Log: ctrl.LoggerFrom(ctx).WithName("controllers").WithName("KongCustomEntity"), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + IngressClassName: c.IngressClassName, + // KongCustomEntities do not accept entities without `kubernetes.io/ingress.class` annotation + // even the controlled ingress class is the default to avoid putting resources not managed in,. + DisableIngressClassLookups: true, + StatusQueue: kubernetesStatusQueue, + }, + }, // --------------------------------------------------------------------------- // Gateway API Controllers // --------------------------------------------------------------------------- diff --git a/internal/manager/featuregates/feature_gates.go b/internal/manager/featuregates/feature_gates.go index 237bfcc8c4..914ae273e6 100644 --- a/internal/manager/featuregates/feature_gates.go +++ b/internal/manager/featuregates/feature_gates.go @@ -32,6 +32,13 @@ const ( // of entity errors returned by the Kong Admin API. FallbackConfiguration = "FallbackConfiguration" + // KongCustomEntity is the name of the feature-gate for enabling KongCustomEntity CR reconciliation + // for configuring custom Kong entities that KIC does not support yet. + // Requires feature gate `FillIDs` to be enabled. + // TODO: enable the feature gate by default when ready: + // https://github.com/Kong/kubernetes-ingress-controller/issues/6124 + KongCustomEntity = "KongCustomEntity" + // DocsURL provides a link to the documentation for feature gates in the KIC repository. DocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md" ) @@ -53,6 +60,11 @@ func New(setupLog logr.Logger, featureGates map[string]bool) (FeatureGates, erro ctrlMap[feature] = enabled } + // KongCustomEntity requires FillIDs to be enabled, because custom entities requires stable IDs to fill in its "foreign" fields. + if ctrlMap.Enabled(KongCustomEntity) && !ctrlMap.Enabled(FillIDsFeature) { + return nil, fmt.Errorf("%s is required if %s is enabled", FillIDsFeature, KongCustomEntity) + } + return ctrlMap, nil } @@ -73,5 +85,6 @@ func GetFeatureGatesDefaults() FeatureGates { KongServiceFacade: false, SanitizeKonnectConfigDumps: true, FallbackConfiguration: false, + KongCustomEntity: false, } } diff --git a/internal/manager/featuregates/feature_gates_test.go b/internal/manager/featuregates/feature_gates_test.go index a4fcdf9e17..416368f466 100644 --- a/internal/manager/featuregates/feature_gates_test.go +++ b/internal/manager/featuregates/feature_gates_test.go @@ -1,10 +1,12 @@ package featuregates import ( + "fmt" "testing" "github.com/go-logr/zapr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -23,11 +25,17 @@ func TestFeatureGates(t *testing.T) { assert.NoError(t, err) assert.True(t, fgs[GatewayAlphaFeature]) + t.Log("Verifying feature gates setup will return error when settings has conflicts") + featureGates = map[string]bool{KongCustomEntity: true, FillIDsFeature: false} + _, err = New(setupLog, featureGates) + require.Error(t, err) + assert.Contains(t, err.Error(), fmt.Sprintf("%s is required if %s is enabled", FillIDsFeature, KongCustomEntity)) + t.Log("Configuring several invalid feature gates options") featureGates = map[string]bool{"invalidGateway": true} t.Log("Verifying feature gates setup results when invalid feature gates options are present") _, err = New(setupLog, featureGates) - assert.Error(t, err) + require.Error(t, err) assert.Contains(t, err.Error(), "invalidGateway is not a valid feature") } diff --git a/internal/manager/run.go b/internal/manager/run.go index 335841972d..8d4f1d9a11 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -64,7 +64,6 @@ func Run( if err != nil { return fmt.Errorf("failed to configure feature gates: %w", err) } - setupLog.Info("Getting the kubernetes client configuration") kubeconfig, err := c.GetKubeconfig() if err != nil { @@ -180,7 +179,7 @@ func Run( referenceIndexers := ctrlref.NewCacheIndexers(setupLog.WithName("reference-indexers")) cache := store.NewCacheStores() storer := store.New(cache, c.IngressClassName, logger) - configTranslator, err := translator.NewTranslator(logger, storer, c.KongWorkspace, translatorFeatureFlags) + configTranslator, err := translator.NewTranslator(logger, storer, c.KongWorkspace, translatorFeatureFlags, NewSchemaServiceGetter(clientsManager)) if err != nil { return fmt.Errorf("failed to create translator: %w", err) } diff --git a/internal/manager/schema_service.go b/internal/manager/schema_service.go new file mode 100644 index 0000000000..183af19a73 --- /dev/null +++ b/internal/manager/schema_service.go @@ -0,0 +1,36 @@ +package manager + +import ( + "github.com/kong/go-kong/kong" + + "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" + "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/translator" +) + +// GatewayClientsProvider is an interface that provides clients for the currently discovered Gateway instances. +type GatewayClientsProvider interface { + GatewayClients() []*adminapi.Client +} + +// SchemaServiceGetter returns schema service of an admin API client if there is any client available. +type SchemaServiceGetter struct { + clientsManager GatewayClientsProvider +} + +// NewSchemaServiceGetter creates a schema service getter that uses given client manager to maintain admin API clients. +func NewSchemaServiceGetter(cm GatewayClientsProvider) SchemaServiceGetter { + return SchemaServiceGetter{ + clientsManager: cm, + } +} + +// GetSchemaService returns the schema service for admin API client. +// It uses the configured clients manager to get the clients and then it uses one of those to obtain the service. +func (ssg SchemaServiceGetter) GetSchemaService() kong.AbstractSchemaService { + clients := ssg.clientsManager.GatewayClients() + if len(clients) > 0 { + return clients[0].AdminAPIClient().Schemas + } + // returns a fake schema service when no gateway clients available. + return translator.UnavailableSchemaService{} +} diff --git a/internal/store/fake_store.go b/internal/store/fake_store.go index ba65dacfc7..142488106f 100644 --- a/internal/store/fake_store.go +++ b/internal/store/fake_store.go @@ -53,6 +53,7 @@ type FakeObjects struct { KongUpstreamPolicies []*kongv1beta1.KongUpstreamPolicy KongServiceFacades []*incubatorv1alpha1.KongServiceFacade KongVaults []*kongv1alpha1.KongVault + KongCustomEntities []*kongv1alpha1.KongCustomEntity } // NewFakeStore creates a store backed by the objects passed in as arguments. @@ -214,6 +215,12 @@ func NewFakeStore( return nil, err } } + kongCustomEntityStore := cache.NewStore(namespacedKeyFunc) + for _, e := range objects.KongCustomEntities { + if err := kongCustomEntityStore.Add(e); err != nil { + return nil, err + } + } s = &Store{ stores: CacheStores{ @@ -240,6 +247,7 @@ func NewFakeStore( KongUpstreamPolicy: kongUpstreamPolicyStore, KongServiceFacade: kongServiceFacade, KongVault: kongVaultStore, + KongCustomEntity: kongCustomEntityStore, }, ingressClass: annotations.DefaultIngressClass, isValidIngressClass: annotations.IngressClassValidatorFuncFromObjectMeta(annotations.DefaultIngressClass), @@ -277,6 +285,7 @@ func (objects FakeObjects) MarshalToYAML() ([]byte, error) { reflect.TypeOf(&kongv1.KongConsumer{}): kongv1.SchemeGroupVersion.WithKind("KongConsumer"), reflect.TypeOf(&kongv1beta1.KongConsumerGroup{}): kongv1beta1.SchemeGroupVersion.WithKind("KongConsumerGroup"), reflect.TypeOf(&kongv1alpha1.KongVault{}): kongv1alpha1.SchemeGroupVersion.WithKind(kongv1alpha1.KongVaultKind), + reflect.TypeOf(&kongv1alpha1.KongCustomEntity{}): kongv1alpha1.SchemeGroupVersion.WithKind(kongv1alpha1.KongCustomEntityKind), } out := &bytes.Buffer{} @@ -320,6 +329,7 @@ func (objects FakeObjects) MarshalToYAML() ([]byte, error) { allObjects = append(allObjects, lo.ToAnySlice(objects.KongConsumers)...) allObjects = append(allObjects, lo.ToAnySlice(objects.KongConsumerGroups)...) allObjects = append(allObjects, lo.ToAnySlice(objects.KongVaults)...) + allObjects = append(allObjects, lo.ToAnySlice(objects.KongCustomEntities)...) for _, obj := range allObjects { if err := fillGVKAndAppendToBuffer(obj.(runtime.Object)); err != nil { diff --git a/internal/store/store.go b/internal/store/store.go index 86972f4818..f7e30bc2db 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -73,6 +73,7 @@ type Storer interface { GetKongUpstreamPolicy(namespace, name string) (*kongv1beta1.KongUpstreamPolicy, error) GetKongServiceFacade(namespace, name string) (*incubatorv1alpha1.KongServiceFacade, error) GetKongVault(name string) (*kongv1alpha1.KongVault, error) + GetKongCustomEntity(namespace, name string) (*kongv1alpha1.KongCustomEntity, error) ListIngressesV1() []*netv1.Ingress ListIngressClassesV1() []*netv1.IngressClass @@ -93,6 +94,7 @@ type Storer interface { ListKongConsumerGroups() []*kongv1beta1.KongConsumerGroup ListCACerts() ([]*corev1.Secret, error) ListKongVaults() []*kongv1alpha1.KongVault + ListKongCustomEntities() []*kongv1alpha1.KongCustomEntity } // Store implements Storer and can be used to list Ingress, Services @@ -562,6 +564,18 @@ func (s Store) GetKongVault(name string) (*kongv1alpha1.KongVault, error) { return p.(*kongv1alpha1.KongVault), nil } +func (s Store) GetKongCustomEntity(namespace, name string) (*kongv1alpha1.KongCustomEntity, error) { + key := fmt.Sprintf("%v/%v", namespace, name) + e, exists, err := s.stores.KongCustomEntity.GetByKey(key) + if err != nil { + return nil, err + } + if !exists { + return nil, NotFoundError{fmt.Sprintf("KongCustomEntity %s/%s not found", namespace, name)} + } + return e.(*kongv1alpha1.KongCustomEntity), nil +} + // ListKongConsumers returns all KongConsumers filtered by the ingress.class // annotation. func (s Store) ListKongConsumers() []*kongv1.KongConsumer { @@ -670,6 +684,21 @@ func (s Store) ListKongVaults() []*kongv1alpha1.KongVault { return kongVaults } +func (s Store) ListKongCustomEntities() []*kongv1alpha1.KongCustomEntity { + var kongCustomEntities []*kongv1alpha1.KongCustomEntity + for _, obj := range s.stores.KongCustomEntity.List() { + kongCustomEntity, ok := obj.(*kongv1alpha1.KongCustomEntity) + if ok && s.isValidIngressClass( + &metav1.ObjectMeta{ + Annotations: map[string]string{annotations.IngressClassKey: kongCustomEntity.Spec.ControllerName}, + }, annotations.IngressClassKey, s.getIngressClassHandling(), + ) { + kongCustomEntities = append(kongCustomEntities, kongCustomEntity) + } + } + return kongCustomEntities +} + // getIngressClassHandling returns annotations.ExactOrEmptyClassMatch if an IngressClass is the default class, or // annotations.ExactClassMatch if the IngressClass is not default or does not exist. func (s Store) getIngressClassHandling() annotations.ClassMatching { @@ -760,6 +789,8 @@ func mkObjFromGVK(gvk schema.GroupVersionKind) (runtime.Object, error) { return &kongv1beta1.KongUpstreamPolicy{}, nil case incubatorv1alpha1.SchemeGroupVersion.WithKind("KongServiceFacade"): return &incubatorv1alpha1.KongServiceFacade{}, nil + case kongv1alpha1.GroupVersion.WithKind(kongv1alpha1.KongCustomEntityKind): + return &kongv1alpha1.KongCustomEntity{}, nil default: return nil, fmt.Errorf("%s is not a supported runtime.Object", gvk) } diff --git a/internal/store/zz_generated_cache_stores.go b/internal/store/zz_generated_cache_stores.go index 2df2e66158..eaf237e7a2 100644 --- a/internal/store/zz_generated_cache_stores.go +++ b/internal/store/zz_generated_cache_stores.go @@ -46,6 +46,7 @@ type CacheStores struct { IngressClassParametersV1alpha1 cache.Store KongServiceFacade cache.Store KongVault cache.Store + KongCustomEntity cache.Store l *sync.RWMutex } @@ -76,6 +77,7 @@ func NewCacheStores() CacheStores { IngressClassParametersV1alpha1: cache.NewStore(namespacedKeyFunc), KongServiceFacade: cache.NewStore(namespacedKeyFunc), KongVault: cache.NewStore(clusterWideKeyFunc), + KongCustomEntity: cache.NewStore(namespacedKeyFunc), l: &sync.RWMutex{}, } @@ -133,6 +135,8 @@ func (c CacheStores) Get(obj runtime.Object) (item interface{}, exists bool, err return c.KongServiceFacade.Get(obj) case *kongv1alpha1.KongVault: return c.KongVault.Get(obj) + case *kongv1alpha1.KongCustomEntity: + return c.KongCustomEntity.Get(obj) } return nil, false, fmt.Errorf("%T is not a supported cache object type", obj) } @@ -190,6 +194,8 @@ func (c CacheStores) Add(obj runtime.Object) error { return c.KongServiceFacade.Add(obj) case *kongv1alpha1.KongVault: return c.KongVault.Add(obj) + case *kongv1alpha1.KongCustomEntity: + return c.KongCustomEntity.Add(obj) } return fmt.Errorf("cannot add unsupported kind %q to the store", obj.GetObjectKind().GroupVersionKind()) } @@ -247,6 +253,8 @@ func (c CacheStores) Delete(obj runtime.Object) error { return c.KongServiceFacade.Delete(obj) case *kongv1alpha1.KongVault: return c.KongVault.Delete(obj) + case *kongv1alpha1.KongCustomEntity: + return c.KongCustomEntity.Delete(obj) } return fmt.Errorf("cannot delete unsupported kind %q from the store", obj.GetObjectKind().GroupVersionKind()) } @@ -277,6 +285,7 @@ func (c CacheStores) ListAllStores() []cache.Store { c.IngressClassParametersV1alpha1, c.KongServiceFacade, c.KongVault, + c.KongCustomEntity, } } @@ -306,5 +315,6 @@ func (c CacheStores) SupportedTypes() []client.Object { &kongv1alpha1.IngressClassParameters{}, &incubatorv1alpha1.KongServiceFacade{}, &kongv1alpha1.KongVault{}, + &kongv1alpha1.KongCustomEntity{}, } } diff --git a/internal/store/zz_generated_cache_stores_test.go b/internal/store/zz_generated_cache_stores_test.go index eae0631ae3..58a66b1bb8 100644 --- a/internal/store/zz_generated_cache_stores_test.go +++ b/internal/store/zz_generated_cache_stores_test.go @@ -139,6 +139,11 @@ func TestCacheStores(t *testing.T) { name: "KongVault", objectToStore: &kongv1alpha1.KongVault{}, }, + + { + name: "KongCustomEntity", + objectToStore: &kongv1alpha1.KongCustomEntity{}, + }, } for _, tc := range testCases { diff --git a/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml b/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml index fe7fef5742..24b46aba63 100644 --- a/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml index a9ec331080..f40da8654a 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-dbless-konnect.yaml b/test/e2e/manifests/all-in-one-dbless-konnect.yaml index ffabcb9190..f0aa713bfe 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-dbless.yaml b/test/e2e/manifests/all-in-one-dbless.yaml index f3c479de2e..c167555156 100644 --- a/test/e2e/manifests/all-in-one-dbless.yaml +++ b/test/e2e/manifests/all-in-one-dbless.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-postgres-enterprise.yaml b/test/e2e/manifests/all-in-one-postgres-enterprise.yaml index a418f204d3..ca7ab78a50 100644 --- a/test/e2e/manifests/all-in-one-postgres-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-postgres-enterprise.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml index 2069d66e1e..8fdc8e9bea 100644 --- a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml +++ b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/e2e/manifests/all-in-one-postgres.yaml b/test/e2e/manifests/all-in-one-postgres.yaml index ce07f1a498..5a8a3e4985 100644 --- a/test/e2e/manifests/all-in-one-postgres.yaml +++ b/test/e2e/manifests/all-in-one-postgres.yaml @@ -710,6 +710,203 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: kongcustomentities.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongCustomEntity + listKind: KongCustomEntityList + plural: kongcustomentities + shortNames: + - kce + singular: kongcustomentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: type of the Kong entity + jsonPath: .spec.type + name: Entity Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Programmed")].status + name: Programmed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: KongCustomEntity defines a "custom" Kong entity that KIC cannot + support the entity type directly. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + controllerName: + description: ControllerName specifies the controller that should reconcile + it, like ingress class. + type: string + fields: + description: Fields defines the fields of the Kong entity itself. + x-kubernetes-preserve-unknown-fields: true + parentRef: + description: |- + ParentRef references the kubernetes resource it attached to when its scope is "attached". + Currently only KongPlugin/KongClusterPlugin allowed. This will make the custom entity to be attached + to the entity(service/route/consumer) where the plugin is attached. + properties: + group: + type: string + kind: + type: string + name: + type: string + namespace: + description: Empty namespace means the same namespace of the owning + object. + type: string + required: + - name + type: object + type: + description: EntityType is the type of the Kong entity. The type is + used in generating declarative configuration. + type: string + required: + - controllerName + - fields + - type + type: object + status: + description: Status stores the reconciling status of the resource. + properties: + conditions: + default: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + description: |- + Conditions describe the current conditions of the KongCustomEntityStatus. + + + Known condition types are: + + + * "Programmed" + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: The spec.type field is immutable + rule: self.spec.type == oldSelf.spec.type + - message: The spec.type field cannot be known Kong entity types + rule: '!(self.spec.type in [''services'',''routes'',''upstreams'',''targets'',''plugins'',''consumers'',''consumer_groups''])' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 @@ -3094,6 +3291,22 @@ rules: - get - patch - update +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongcustomentities/status + verbs: + - get + - patch + - update - apiGroups: - configuration.konghq.com resources: diff --git a/test/envtest/admission_webhook_envtest_test.go b/test/envtest/admission_webhook_envtest_test.go index d13deb2779..76a0959645 100644 --- a/test/envtest/admission_webhook_envtest_test.go +++ b/test/envtest/admission_webhook_envtest_test.go @@ -1296,7 +1296,8 @@ func TestAdmissionWebhook_KongCustomEntities(t *testing.T) { }, }, }, - valid: true, + requireEnterpriseLicense: true, + valid: true, }, { name: "invalid degraphql_route entity", @@ -1312,7 +1313,8 @@ func TestAdmissionWebhook_KongCustomEntities(t *testing.T) { }, }, }, - valid: false, + requireEnterpriseLicense: true, + valid: false, }, { name: "KongCustomEntity not controlled by the current controller", diff --git a/test/envtest/telemetry_test.go b/test/envtest/telemetry_test.go index a3aa8d31c4..05754f80a8 100644 --- a/test/envtest/telemetry_test.go +++ b/test/envtest/telemetry_test.go @@ -360,6 +360,7 @@ func verifyTelemetryReport(t *testing.T, k8sVersion *version.Info, report string "feature-fillids=true;"+ "feature-gateway-service-discovery=false;"+ "feature-gatewayalpha=false;"+ + "feature-kongcustomentity=false;"+ "feature-kongservicefacade=false;"+ "feature-konnect-sync=false;"+ "feature-rewriteuris=false;"+ From dd00c07db385406ab3a561b234df9d7bf7f3eae9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:30:06 +0200 Subject: [PATCH 34/48] chore(deps): bump google.golang.org/api from 0.182.0 to 0.183.0 (#6140) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.182.0 to 0.183.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.182.0...v0.183.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 2f0b760bfe..482bd32269 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 go.uber.org/zap v1.27.0 - google.golang.org/api v0.182.0 + google.golang.org/api v0.183.0 k8s.io/api v0.30.1 k8s.io/apiextensions-apiserver v0.30.1 k8s.io/apimachinery v0.30.1 @@ -64,7 +64,7 @@ require ( ) require ( - cloud.google.com/go/auth v0.4.2 // indirect + cloud.google.com/go/auth v0.5.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect @@ -205,7 +205,7 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.20.0 // indirect @@ -213,8 +213,8 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index a84512c53c..ea4ca18773 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= -cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= -cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= @@ -514,8 +514,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -583,17 +583,17 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= -google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= From 7de426cbc2aad81f26851b18cd70615b5b03ebac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:30:26 +0200 Subject: [PATCH 35/48] chore(deps): bump golang from 1.22.3 to 1.22.4 (#6141) Bumps golang from 1.22.3 to 1.22.4. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.debug | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d66b7465a..2cbd4a0e64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ### Standard binary # Build the manager binary -FROM --platform=$BUILDPLATFORM golang:1.22.3 as builder +FROM --platform=$BUILDPLATFORM golang:1.22.4 as builder ARG GOPATH ARG GOCACHE diff --git a/Dockerfile.debug b/Dockerfile.debug index 13dbb328a4..e0fe65aed3 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -1,5 +1,5 @@ # Build a manager binary with debug symbols and download Delve -FROM --platform=$BUILDPLATFORM golang:1.22.3 as builder +FROM --platform=$BUILDPLATFORM golang:1.22.4 as builder ARG GOPATH ARG GOCACHE @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=$GOPATH/pkg/mod \ ### Debug # Create an image that runs a debug build with Delve installed -FROM golang:1.22.3 AS debug +FROM golang:1.22.4 AS debug RUN go install github.com/go-delve/delve/cmd/dlv@latest # We want all source so Delve file location operations work COPY --from=builder /workspace/bin/manager-debug / From 21ad6608c4dcd6367c12189938920812b17cabc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 6 Jun 2024 02:22:44 +0200 Subject: [PATCH 36/48] fix: do not sanitize plugins' config (#6138) * fix: do not sanitize plugins' config This reverts commit 2d615e3df5956d6ebd4e0b0137ff6dffc20294c1. * ensure we do not sanitize plugins' config in ut --- CHANGELOG.md | 10 +- internal/dataplane/kongstate/kongstate.go | 4 +- .../dataplane/kongstate/kongstate_test.go | 10 +- internal/dataplane/kongstate/plugin.go | 126 +- internal/dataplane/kongstate/plugin_test.go | 1034 ++++++++--------- internal/dataplane/kongstate/types.go | 14 + 6 files changed, 495 insertions(+), 703 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b12ea1c8..8b022708ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -211,15 +211,17 @@ Adding a new version? You'll need three changes: is seen as a parent other than the controller and ignored in parentRef check. [#5919](https://github.com/Kong/kubernetes-ingress-controller/pull/5919) - Redacted values no longer cause collisions in configuration reported to Konnect. - [5964](https://github.com/Kong/kubernetes-ingress-controller/pull/5964) + [#5964](https://github.com/Kong/kubernetes-ingress-controller/pull/5964) - The `--dump-sensitive-config` flag is no longer backwards. - [6073](https://github.com/Kong/kubernetes-ingress-controller/pull/6073) + [#6073](https://github.com/Kong/kubernetes-ingress-controller/pull/6073) - Fixed KIC clearing Gateway API *Route status of routes that it shouldn't reconcilce, e.g. those attached to Gateways that do not belong to GatewayClass that KIC reconciles. - [6079](https://github.com/Kong/kubernetes-ingress-controller/pull/6079) + [#6079](https://github.com/Kong/kubernetes-ingress-controller/pull/6079) - Fixed KIC non leaders correctly getting up to date Admin API addresses by not requiring leader election for the related controller. - [6126](https://github.com/Kong/kubernetes-ingress-controller/pull/6126) + [#6126](https://github.com/Kong/kubernetes-ingress-controller/pull/6126) +- KongPlugin's `config` field is no longer incorrectly sanitized. + [#6138](https://github.com/Kong/kubernetes-ingress-controller/pull/6138) ### Changed diff --git a/internal/dataplane/kongstate/kongstate.go b/internal/dataplane/kongstate/kongstate.go index b9c3bbb0be..404e53674b 100644 --- a/internal/dataplane/kongstate/kongstate.go +++ b/internal/dataplane/kongstate/kongstate.go @@ -55,9 +55,7 @@ func (ks *KongState) SanitizedCopy(uuidGenerator util.UUIDGenerator) *KongState return }(), CACertificates: ks.CACertificates, - Plugins: lo.Map(ks.Plugins, func(p Plugin, _ int) Plugin { - return p.SanitizedCopy() - }), + Plugins: ks.Plugins, Consumers: func() (res []Consumer) { for _, v := range ks.Consumers { res = append(res, *v.SanitizedCopy(uuidGenerator)) diff --git a/internal/dataplane/kongstate/kongstate_test.go b/internal/dataplane/kongstate/kongstate_test.go index 83f42514e4..6ab9c89c64 100644 --- a/internal/dataplane/kongstate/kongstate_test.go +++ b/internal/dataplane/kongstate/kongstate_test.go @@ -60,10 +60,7 @@ func TestKongState_SanitizedCopy(t *testing.T) { Upstreams: []Upstream{{Upstream: kong.Upstream{ID: kong.String("1")}}}, Certificates: []Certificate{{Certificate: kong.Certificate{ID: kong.String("1"), Key: kong.String("secret")}}}, CACertificates: []kong.CACertificate{{ID: kong.String("1")}}, - Plugins: []Plugin{{ - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{WholeConfigIsSensitive: true}, - Plugin: kong.Plugin{ID: kong.String("1"), Config: kong.Configuration{"secret": "secretValue"}}, - }}, + Plugins: []Plugin{{Plugin: kong.Plugin{ID: kong.String("1"), Config: map[string]interface{}{"key": "secret"}}}}, Consumers: []Consumer{{ KeyAuths: []*KeyAuth{{kong.KeyAuth{ID: kong.String("1"), Key: kong.String("secret")}}}, }}, @@ -103,10 +100,7 @@ func TestKongState_SanitizedCopy(t *testing.T) { Upstreams: []Upstream{{Upstream: kong.Upstream{ID: kong.String("1")}}}, Certificates: []Certificate{{Certificate: kong.Certificate{ID: kong.String("1"), Key: redactedString}}}, CACertificates: []kong.CACertificate{{ID: kong.String("1")}}, - Plugins: []Plugin{{ - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{WholeConfigIsSensitive: true}, - Plugin: kong.Plugin{ID: kong.String("1"), Config: kong.Configuration{"secret": *redactedString}}, - }}, + Plugins: []Plugin{{Plugin: kong.Plugin{ID: kong.String("1"), Config: map[string]interface{}{"key": "secret"}}}}, // We don't redact plugins' config. Consumers: []Consumer{{ KeyAuths: []*KeyAuth{{kong.KeyAuth{ID: kong.String("1"), Key: kong.String("{vault://52fdfc07-2182-454f-963f-5f0f9a621d72}")}}}, }}, diff --git a/internal/dataplane/kongstate/plugin.go b/internal/dataplane/kongstate/plugin.go index 2305f4931f..31ce8c5824 100644 --- a/internal/dataplane/kongstate/plugin.go +++ b/internal/dataplane/kongstate/plugin.go @@ -4,14 +4,11 @@ import ( "encoding/json" "errors" "fmt" - "strings" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/kong/go-kong/kong" - "github.com/samber/lo" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" @@ -19,107 +16,6 @@ import ( kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1" ) -// Plugin represents a plugin Object in Kong. -type Plugin struct { - kong.Plugin - K8sParent client.Object - SensitiveFieldsMeta PluginSensitiveFieldsMetadata -} - -func (p Plugin) DeepCopy() Plugin { - return Plugin{ - Plugin: *p.Plugin.DeepCopy(), - K8sParent: p.K8sParent, - SensitiveFieldsMeta: p.SensitiveFieldsMeta, - } -} - -func (p Plugin) SanitizedCopy() Plugin { - // We do not want to return an error if any of below fails - the best we can do - // is to return a plugin with wholly redacted config. - // Let's have a closure returning a plugin with wholly redacted config prepared. - whollySanitized := func() Plugin { - p := p.DeepCopy() - p.Config = sanitizeWholePluginConfig(p.Config) - return p - } - - // If the whole config is sensitive, we need to redact the entire config. - if p.SensitiveFieldsMeta.WholeConfigIsSensitive { - return whollySanitized() - } - - // If there are JSON paths, we need to redact them. - if len(p.SensitiveFieldsMeta.JSONPaths) > 0 { - var patchOperations []string - for _, path := range p.SensitiveFieldsMeta.JSONPaths { - // If the path is empty, we need to sanitize the whole config. - // An empty path means that the patch is on the root of the config. - if path == "" { - return whollySanitized() - } - - patchOperations = append(patchOperations, fmt.Sprintf( - `{"op":"replace","path":"%s","value":"%s"}`, - path, - *redactedString, - )) - } - - // Decode the patch and apply it to the config. - // We need to marshal the config to JSON and then unmarshal it back to Configuration - // because the patch library works with bytes. - patch, err := jsonpatch.DecodePatch([]byte(fmt.Sprintf("[%s]", strings.Join(patchOperations, ",")))) - if err != nil { - return whollySanitized() - } - configB, err := json.Marshal(p.Config) - if err != nil { - return whollySanitized() - } - sanitizedConfigB, err := patch.Apply(configB) - if err != nil { - return whollySanitized() - } - sanitizedConfig := kong.Configuration{} - if err := json.Unmarshal(sanitizedConfigB, &sanitizedConfig); err != nil { - return whollySanitized() - } - - sanitized := p.DeepCopy() - sanitized.Config = sanitizedConfig - return sanitized - } - - // Nothing to sanitize. - return p -} - -// sanitizeWholePluginConfig redacts the entire config of a plugin by replacing all of its -// values with a redacted string. -func sanitizeWholePluginConfig(config kong.Configuration) kong.Configuration { - sanitized := config.DeepCopy() - for k := range config { - sanitized[k] = *redactedString - } - return sanitized -} - -// PluginSensitiveFieldsMetadata holds metadata about sensitive fields in a plugin's configuration. -// It can be used to sanitize them before exposing the configuration to the user (e.g. in debug dumps -// or in Konnect Admin API). -type PluginSensitiveFieldsMetadata struct { - // WholeConfigIsSensitive indicates that the entire configuration of the plugin is sensitive. - // If this is true, the configuration should be redacted entirely (each of its fields' values - // should be replaced with a redacted string). - WholeConfigIsSensitive bool - - // JSONPaths holds a list of JSON paths to sensitive fields in the plugin's configuration. - // If this is not empty, the configuration should be redacted by replacing the values of the - // fields at these paths with a redacted string. - JSONPaths []string -} - // getKongPluginOrKongClusterPlugin fetches a KongPlugin or KongClusterPlugin (as fallback) from the store. // If both are not found, an error is returned. func getKongPluginOrKongClusterPlugin(s store.Storer, namespace, name string) ( @@ -181,14 +77,6 @@ func kongPluginFromK8SClusterPlugin( } } - // Prepare sensitive fields metadata for the plugin. - sensitiveFieldsMeta := PluginSensitiveFieldsMetadata{ - JSONPaths: lo.Map(k8sPlugin.ConfigPatches, func(patch kongv1.NamespacedConfigPatch, _ int) string { - return patch.Path - }), - WholeConfigIsSensitive: k8sPlugin.ConfigFrom != nil, - } - return Plugin{ Plugin: plugin{ Name: k8sPlugin.PluginName, @@ -201,8 +89,7 @@ func kongPluginFromK8SClusterPlugin( Protocols: protocolsToStrings(k8sPlugin.Protocols), Tags: util.GenerateTagsForObject(&k8sPlugin), }.toKongPlugin(), - K8sParent: &k8sPlugin, - SensitiveFieldsMeta: sensitiveFieldsMeta, + K8sParent: &k8sPlugin, }, nil } @@ -244,14 +131,6 @@ func kongPluginFromK8SPlugin( } } - // Prepare sensitive fields metadata for the plugin. - sensitiveFieldsMeta := PluginSensitiveFieldsMetadata{ - JSONPaths: lo.Map(k8sPlugin.ConfigPatches, func(patch kongv1.ConfigPatch, _ int) string { - return patch.Path - }), - WholeConfigIsSensitive: k8sPlugin.ConfigFrom != nil, - } - return Plugin{ Plugin: plugin{ Name: k8sPlugin.PluginName, @@ -264,8 +143,7 @@ func kongPluginFromK8SPlugin( Protocols: protocolsToStrings(k8sPlugin.Protocols), Tags: util.GenerateTagsForObject(&k8sPlugin), }.toKongPlugin(), - K8sParent: &k8sPlugin, - SensitiveFieldsMeta: sensitiveFieldsMeta, + K8sParent: &k8sPlugin, }, nil } diff --git a/internal/dataplane/kongstate/plugin_test.go b/internal/dataplane/kongstate/plugin_test.go index 5faf399c26..fed7b3f1cb 100644 --- a/internal/dataplane/kongstate/plugin_test.go +++ b/internal/dataplane/kongstate/plugin_test.go @@ -5,7 +5,6 @@ import ( "github.com/kong/go-kong/kong" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,6 +14,7 @@ import ( ) func TestKongPluginFromK8SClusterPlugin(t *testing.T) { + assert := assert.New(t) store, _ := store.NewFakeStore(store.FakeObjects{ Secrets: []*corev1.Secret{ { @@ -32,284 +32,277 @@ func TestKongPluginFromK8SClusterPlugin(t *testing.T) { }, }, }) - + type args struct { + plugin kongv1.KongClusterPlugin + } tests := []struct { name string - plugin kongv1.KongClusterPlugin - want Plugin + args args + want kong.Plugin wantErr bool }{ { name: "basic configuration", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - InstanceName: "example", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + InstanceName: "example", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - }, - Protocols: kong.StringSlice("http"), - InstanceName: kong.String("example"), + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{JSONPaths: []string{}}, + Protocols: kong.StringSlice("http"), + InstanceName: kong.String("example"), }, wantErr: false, }, { name: "secret configuration", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - ConfigFrom: &kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Key: "correlation-id-config", - Secret: "conf-secret", - Namespace: "default", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + ConfigFrom: &kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Key: "correlation-id-config", + Secret: "conf-secret", + Namespace: "default", + }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - WholeConfigIsSensitive: true, - JSONPaths: []string{}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, + Protocols: kong.StringSlice("http"), }, wantErr: false, }, { name: "missing secret configuration", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - ConfigFrom: &kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Key: "correlation-id-config", - Secret: "missing", - Namespace: "default", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + ConfigFrom: &kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Key: "correlation-id-config", + Secret: "missing", + Namespace: "default", + }, }, }, }, - want: Plugin{}, + want: kong.Plugin{}, wantErr: true, }, { name: "non-JSON configuration", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{{}`), + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{{}`), + }, }, }, - want: Plugin{}, + want: kong.Plugin{}, wantErr: true, }, { name: "both Config and ConfigFrom set", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigFrom: &kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Key: "correlation-id-config", - Secret: "conf-secret", - Namespace: "default", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigFrom: &kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Key: "correlation-id-config", + Secret: "conf-secret", + Namespace: "default", + }, }, }, }, - want: Plugin{}, + want: kong.Plugin{}, wantErr: true, }, { name: "Config and ConfigPatches set", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Key: "correlation-id-generator", - Secret: "conf-secret", - Namespace: "default", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Key: "correlation-id-generator", + Secret: "conf-secret", + Namespace: "default", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - "generator": "uuid", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/generator"}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", + "generator": "uuid", }, + Protocols: kong.StringSlice("http"), }, }, { name: "configPatch on subpath of non-exist path", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "response-transformer", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"replace":{"headers":["foo:bar"]}}`), - }, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/add/headers", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "response-transformer-add-headers", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "response-transformer", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"replace":{"headers":["foo:bar"]}}`), + }, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/add/headers", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "response-transformer-add-headers", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("response-transformer"), - Config: kong.Configuration{ - "replace": map[string]interface{}{ - "headers": []interface{}{ - "foo:bar", - }, + want: kong.Plugin{ + Name: kong.String("response-transformer"), + Config: kong.Configuration{ + "replace": map[string]interface{}{ + "headers": []interface{}{ + "foo:bar", }, - "add": map[string]interface{}{ - "headers": []interface{}{ - "h1:v1", - "h2:v2", - }, + }, + "add": map[string]interface{}{ + "headers": []interface{}{ + "h1:v1", + "h2:v2", }, }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/add/headers"}, }, + Protocols: kong.StringSlice("http"), }, }, { name: "empty config and configPatch for particular paths", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{}, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/header_name", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-headername", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{}, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/header_name", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-headername", + Secret: "conf-secret", + }, }, }, - }, - { - Path: "/generator", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-generator", - Secret: "conf-secret", + { + Path: "/generator", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-generator", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - "generator": "uuid", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/header_name", "/generator"}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", + "generator": "uuid", }, + Protocols: kong.StringSlice("http"), }, }, { name: "empty config and configPatch for whole object", - plugin: kongv1.KongClusterPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{}, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-config", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{}, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-config", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{""}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, + Protocols: kong.StringSlice("http"), }, }, { name: "missing secret in configPatches", - plugin: kongv1.KongClusterPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-generator", - Secret: "missing-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-generator", + Secret: "missing-secret", + }, }, }, }, @@ -319,23 +312,25 @@ func TestKongPluginFromK8SClusterPlugin(t *testing.T) { }, { name: "missing key of secret in cofigPatches", - plugin: kongv1.KongClusterPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-missing", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-missing", + Secret: "conf-secret", + }, }, }, }, @@ -345,23 +340,25 @@ func TestKongPluginFromK8SClusterPlugin(t *testing.T) { }, { name: "invalid value in configPatches", - plugin: kongv1.KongClusterPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.NamespacedConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.NamespacedConfigSource{ - SecretValue: kongv1.NamespacedSecretValueFromSource{ - Namespace: "default", - Key: "correlation-id-invalid", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongClusterPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.NamespacedConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.NamespacedConfigSource{ + SecretValue: kongv1.NamespacedSecretValueFromSource{ + Namespace: "default", + Key: "correlation-id-invalid", + Secret: "conf-secret", + }, }, }, }, @@ -372,18 +369,19 @@ func TestKongPluginFromK8SClusterPlugin(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := kongPluginFromK8SClusterPlugin(store, tt.plugin) - if tt.wantErr { - require.Error(t, err) + got, err := kongPluginFromK8SClusterPlugin(store, tt.args.plugin) + if (err != nil) != tt.wantErr { + t.Errorf("kongPluginFromK8SClusterPlugin error = %v, wantErr %v", err, tt.wantErr) return } - tt.want.K8sParent = tt.plugin.DeepCopy() - assert.Equal(t, tt.want, got) + assert.Equal(tt.want, got.Plugin) + assert.NotEmpty(t, got.K8sParent) }) } } func TestKongPluginFromK8SPlugin(t *testing.T) { + assert := assert.New(t) store, _ := store.NewFakeStore(store.FakeObjects{ Secrets: []*corev1.Secret{ { @@ -401,296 +399,293 @@ func TestKongPluginFromK8SPlugin(t *testing.T) { }, }, }) + type args struct { + plugin kongv1.KongPlugin + } tests := []struct { name string - plugin kongv1.KongPlugin - want Plugin + args args + want kong.Plugin wantErr bool }{ { name: "basic configuration", - plugin: kongv1.KongPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - InstanceName: "example", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), + args: args{ + plugin: kongv1.KongPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + InstanceName: "example", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - }, - Protocols: kong.StringSlice("http"), - InstanceName: kong.String("example"), + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{JSONPaths: []string{}}, + Protocols: kong.StringSlice("http"), + InstanceName: kong.String("example"), }, wantErr: false, }, { name: "secret configuration", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - ConfigFrom: &kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-config", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", }, - }, - }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + ConfigFrom: &kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-config", + Secret: "conf-secret", + }, }, - Protocols: kong.StringSlice("http"), }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - WholeConfigIsSensitive: true, - JSONPaths: []string{}, + }, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, + Protocols: kong.StringSlice("http"), }, wantErr: false, }, { name: "missing secret configuration", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - ConfigFrom: &kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-config", - Secret: "missing", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + ConfigFrom: &kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-config", + Secret: "missing", + }, }, }, }, + want: kong.Plugin{}, wantErr: true, }, { name: "non-JSON configuration", - plugin: kongv1.KongPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{{}`), + args: args{ + plugin: kongv1.KongPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{{}`), + }, }, }, + want: kong.Plugin{}, wantErr: true, }, { name: "both Config and ConfigFrom set", - plugin: kongv1.KongPlugin{ - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigFrom: &kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-config", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigFrom: &kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-config", + Secret: "conf-secret", + }, }, }, }, + want: kong.Plugin{}, wantErr: true, }, { name: "config and configPatches set", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-generator", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-generator", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - "generator": "uuid", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/generator"}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", + "generator": "uuid", }, + Protocols: kong.StringSlice("http"), }, }, { name: "configPatch on subpath of non-exist path", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "response-transformer", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"replace":{"headers":["foo:bar"]}}`), - }, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/add/headers", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "response-transformer-add-headers", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "response-transformer", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"replace":{"headers":["foo:bar"]}}`), + }, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/add/headers", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "response-transformer-add-headers", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("response-transformer"), - Config: kong.Configuration{ - "replace": map[string]interface{}{ - "headers": []interface{}{ - "foo:bar", - }, + want: kong.Plugin{ + Name: kong.String("response-transformer"), + Config: kong.Configuration{ + "replace": map[string]interface{}{ + "headers": []interface{}{ + "foo:bar", }, - "add": map[string]interface{}{ - "headers": []interface{}{ - "h1:v1", - "h2:v2", - }, + }, + "add": map[string]interface{}{ + "headers": []interface{}{ + "h1:v1", + "h2:v2", }, }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/add/headers"}, }, + Protocols: kong.StringSlice("http"), }, }, { name: "empty config and configPatch for particular paths", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{}, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/header_name", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-headername", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{}, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/header_name", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-headername", + Secret: "conf-secret", + }, }, }, - }, - { - Path: "/generator", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-generator", - Secret: "conf-secret", + { + Path: "/generator", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-generator", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - "generator": "uuid", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{"/header_name", "/generator"}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", + "generator": "uuid", }, + Protocols: kong.StringSlice("http"), }, }, { name: "empty config and configPatch for whole object", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{}, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-config", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{}, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-config", + Secret: "conf-secret", + }, }, }, }, }, }, - want: Plugin{ - Plugin: kong.Plugin{ - Name: kong.String("correlation-id"), - Config: kong.Configuration{ - "header_name": "foo", - }, - Protocols: kong.StringSlice("http"), - }, - SensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{""}, + want: kong.Plugin{ + Name: kong.String("correlation-id"), + Config: kong.Configuration{ + "header_name": "foo", }, + Protocols: kong.StringSlice("http"), }, }, { name: "missing secret in configPatches", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-generator", - Secret: "missing-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-generator", + Secret: "missing-secret", + }, }, }, }, @@ -700,23 +695,25 @@ func TestKongPluginFromK8SPlugin(t *testing.T) { }, { name: "missing key of secret in configPatches", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-missing", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-missing", + Secret: "conf-secret", + }, }, }, }, @@ -726,23 +723,25 @@ func TestKongPluginFromK8SPlugin(t *testing.T) { }, { name: "invalid value in configPatches", - plugin: kongv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Protocols: []kongv1.KongProtocol{"http"}, - PluginName: "correlation-id", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"header_name": "foo"}`), - }, - ConfigPatches: []kongv1.ConfigPatch{ - { - Path: "/generator", - ValueFrom: kongv1.ConfigSource{ - SecretValue: kongv1.SecretValueFromSource{ - Key: "correlation-id-invalid", - Secret: "conf-secret", + args: args{ + plugin: kongv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Protocols: []kongv1.KongProtocol{"http"}, + PluginName: "correlation-id", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"header_name": "foo"}`), + }, + ConfigPatches: []kongv1.ConfigPatch{ + { + Path: "/generator", + ValueFrom: kongv1.ConfigSource{ + SecretValue: kongv1.SecretValueFromSource{ + Key: "correlation-id-invalid", + Secret: "conf-secret", + }, }, }, }, @@ -753,108 +752,15 @@ func TestKongPluginFromK8SPlugin(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := kongPluginFromK8SPlugin(store, tt.plugin) - if tt.wantErr { - require.Error(t, err) + got, err := kongPluginFromK8SPlugin(store, tt.args.plugin) + if (err != nil) != tt.wantErr { + t.Errorf("kongPluginFromK8SPlugin error = %v, wantErr %v", err, tt.wantErr) return } // don't care about tags in this test got.Tags = nil - tt.want.K8sParent = tt.plugin.DeepCopy() - assert.Equal(t, tt.want, got) - }) - } -} - -func TestPlugin_SanitizedCopy(t *testing.T) { - testCases := []struct { - name string - config kong.Configuration - sensitiveFieldsMeta PluginSensitiveFieldsMetadata - expectedSanitizedConfig kong.Configuration - }{ - { - name: "sensitive fields are redacted with JSONPaths", - config: kong.Configuration{ - "secret": "secret-value", - "object": map[string]interface{}{ - "secretObjectField": "secret-object-field-value", - }, - }, - sensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{ - "/secret", - "/object/secretObjectField", - }, - }, - expectedSanitizedConfig: kong.Configuration{ - "secret": "{vault://redacted-value}", - "object": map[string]interface{}{ - "secretObjectField": "{vault://redacted-value}", - }, - }, - }, - { - name: "invalid JSONPath doesn't panic and redacts whole config as fallback", - config: kong.Configuration{ - "secret": "secret-value", - "object": map[string]interface{}{ - "secretObjectField": "secret-object-field-value", - }, - }, - sensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{ - "/not-existing-path", - }, - }, - expectedSanitizedConfig: kong.Configuration{ - "secret": "{vault://redacted-value}", - "object": "{vault://redacted-value}", - }, - }, - { - name: "whole config to sanitize", - config: kong.Configuration{ - "secret": "secret-value", - "object": map[string]interface{}{ - "secretObjectField": "secret-object-field-value", - }, - }, - sensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - WholeConfigIsSensitive: true, - }, - expectedSanitizedConfig: kong.Configuration{ - "secret": "{vault://redacted-value}", - "object": "{vault://redacted-value}", - }, - }, - { - name: "single empty JSON path - whole config is redacted", - config: kong.Configuration{ - "secret": "secret-value", - "object": map[string]interface{}{ - "secretObjectField": "secret-object-field-value", - }, - }, - sensitiveFieldsMeta: PluginSensitiveFieldsMetadata{ - JSONPaths: []string{""}, - }, - expectedSanitizedConfig: kong.Configuration{ - "secret": "{vault://redacted-value}", - "object": "{vault://redacted-value}", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - p := Plugin{ - Plugin: kong.Plugin{ - Config: tc.config, - }, - SensitiveFieldsMeta: tc.sensitiveFieldsMeta, - } - sanitized := p.SanitizedCopy() - assert.Equal(t, tc.expectedSanitizedConfig, sanitized.Config) + assert.Equal(tt.want, got.Plugin) + assert.NotEmpty(t, got.K8sParent) }) } } diff --git a/internal/dataplane/kongstate/types.go b/internal/dataplane/kongstate/types.go index 282d81081f..fcfa7afe5a 100644 --- a/internal/dataplane/kongstate/types.go +++ b/internal/dataplane/kongstate/types.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/kong/go-kong/kong" + "sigs.k8s.io/controller-runtime/pkg/client" ) type PortMode int @@ -65,3 +66,16 @@ func (c *Certificate) SanitizedCopy() *Certificate { }, } } + +// Plugin represents a plugin Object in Kong. +type Plugin struct { + kong.Plugin + K8sParent client.Object +} + +func (p Plugin) DeepCopy() Plugin { + return Plugin{ + Plugin: *p.Plugin.DeepCopy(), + K8sParent: p.K8sParent, + } +} From 09e039df5e6aca25b2947859713e589cdfd1606d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 09:54:06 +0200 Subject: [PATCH 37/48] chore(deps): update dependency kubernetes-sigs/controller-runtime to v0.18.4 (#6143) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .tools_versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tools_versions.yaml b/.tools_versions.yaml index a0b56dd1e8..ae4004bdfe 100644 --- a/.tools_versions.yaml +++ b/.tools_versions.yaml @@ -9,7 +9,7 @@ golangci-lint: "1.59.0" # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold skaffold: "2.12.0" # renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime -setup-envtest: "0.18.3" +setup-envtest: "0.18.4" # renovate: datasource=github-releases depName=elastic/crd-ref-docs crd-ref-docs: "0.0.12" # renovate: datasource=github-releases depName=mikefarah/yq From fc74077903ef2dc052332de216faf2f00efa14a9 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Thu, 6 Jun 2024 17:56:31 +0800 Subject: [PATCH 38/48] conformance: add support for GRPCRoute (#5776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * set grpc as default protocol * skip unsupported case. * add TODO about test case conflicts Signed-off-by: Jintao Zhang Co-authored-by: Patryk Małek --- CHANGELOG.md | 5 ++ examples/gateway-grpcroute-via-http.yaml | 2 +- examples/gateway-grpcroute-via-https.yaml | 2 +- .../gateway/grpcroute_controller.go | 2 +- .../gateway/route_parent_status.go | 5 ++ .../subtranslator/grpcroute_test.go | 18 +++--- .../translator/translate_grpcroute.go | 20 +++++-- test/conformance/gateway_conformance_test.go | 18 ++++++ test/conformance/suite_test.go | 4 ++ test/consts.go | 4 +- test/integration/isolated/grpc_test.go | 58 +++++-------------- test/integration/isolated/ingress_test.go | 14 ++--- test/internal/helpers/gatewayapi.go | 4 +- 13 files changed, 81 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b022708ee..d01a87bef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,9 @@ Adding a new version? You'll need three changes: performance benefits, however, so labeling plugin configuration Secrets and enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) +- Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway. + If you don't set the protocol for Service via `konghq.com/protocol` annotation, Kong will use `grpc` instead of `grpcs`. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) - The `/debug/config/failed` and `/debug/config/successful` diagnostic endpoints now nest configuration dumps under a `config` key. These endpoints previously returned the configuration dump at the root. They now return @@ -192,6 +195,8 @@ Adding a new version? You'll need three changes: [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965) - Fallback configuration no longer omits licenses and vaults. [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) +- Add support for Gateway API GRPCRoute and pass related Gateway API conformance test. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Fixed diff --git a/examples/gateway-grpcroute-via-http.yaml b/examples/gateway-grpcroute-via-http.yaml index bc7ebe3839..50b61c7999 100644 --- a/examples/gateway-grpcroute-via-http.yaml +++ b/examples/gateway-grpcroute-via-http.yaml @@ -58,7 +58,7 @@ spec: protocol: HTTP port: 80 --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-http diff --git a/examples/gateway-grpcroute-via-https.yaml b/examples/gateway-grpcroute-via-https.yaml index 3317bfc837..12e0871ec3 100644 --- a/examples/gateway-grpcroute-via-https.yaml +++ b/examples/gateway-grpcroute-via-https.yaml @@ -70,7 +70,7 @@ spec: certificateRefs: - name: grpcroute-example --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-https diff --git a/internal/controllers/gateway/grpcroute_controller.go b/internal/controllers/gateway/grpcroute_controller.go index 0b5cf7de7c..c9906c53a4 100644 --- a/internal/controllers/gateway/grpcroute_controller.go +++ b/internal/controllers/gateway/grpcroute_controller.go @@ -475,7 +475,7 @@ func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont // if the reference already exists and doesn't require any changes // then just leave it alone. - parentRefKey := gateway.gateway.Namespace + "/" + gateway.gateway.Name + parentRefKey := fmt.Sprintf("%s/%s/%s", gateway.gateway.Namespace, gateway.gateway.Name, gateway.listenerName) if existingGatewayParentStatus, exists := parentStatuses[parentRefKey]; exists { // check if the parentRef and controllerName are equal, and whether the new condition is present in existing conditions if reflect.DeepEqual(existingGatewayParentStatus.ParentRef, gatewayParentStatus.ParentRef) && diff --git a/internal/controllers/gateway/route_parent_status.go b/internal/controllers/gateway/route_parent_status.go index 90719a8fee..f369f8f7d3 100644 --- a/internal/controllers/gateway/route_parent_status.go +++ b/internal/controllers/gateway/route_parent_status.go @@ -46,6 +46,11 @@ func routeParentStatusKey[routeT gatewayapi.RouteT]( namespace, parentRef.GetName(), parentRef.GetSectionName().OrEmpty()) + case *gatewayapi.GRPCRoute: + return fmt.Sprintf("%s/%s/%s", + namespace, + parentRef.GetName(), + parentRef.GetSectionName().OrEmpty()) default: return fmt.Sprintf("%s/%s", namespace, parentRef.GetName()) } diff --git a/internal/dataplane/translator/subtranslator/grpcroute_test.go b/internal/dataplane/translator/subtranslator/grpcroute_test.go index ada1c15518..246aef4441 100644 --- a/internal/dataplane/translator/subtranslator/grpcroute_test.go +++ b/internal/dataplane/translator/subtranslator/grpcroute_test.go @@ -16,13 +16,13 @@ import ( var grpcRouteGVK = schema.GroupVersionKind{ Group: "gateway.networking.k8s.io", - Version: "v1alpha2", + Version: "v1", Kind: "GRPCRoute", } var grpcRouteTypeMeta = metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", } func makeTestGRPCRoute( @@ -33,7 +33,7 @@ func makeTestGRPCRoute( return &gatewayapi.GRPCRoute{ TypeMeta: metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -98,7 +98,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -139,7 +139,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -194,7 +194,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -214,7 +214,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -243,7 +243,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -270,7 +270,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), Paths: kong.StringSlice("/"), }, diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index 2487333df1..f88407d3be 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -53,10 +53,9 @@ func (t *Translator) ingressRulesFromGRPCRoute(result *ingressRules, grpcroute * // each rule may represent a different set of backend services that will be accepting // traffic, so we make separate routes and Kong services for every present rule. for ruleNumber, rule := range spec.Rules { - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. service, err := generateKongServiceFromBackendRefWithRuleNumber( - t.logger, t.storer, result, grpcroute, ruleNumber, "grpcs", grpcBackendRefsToBackendRefs(rule.BackendRefs)..., + t.logger, t.storer, result, grpcroute, ruleNumber, t.getProtocolForKongService(grpcroute), grpcBackendRefsToBackendRefs(rule.BackendRefs)..., ) if err != nil { return err @@ -116,15 +115,14 @@ func (t *Translator) ingressRulesFromGRPCRouteWithPriority( serviceName := subtranslator.KongServiceNameFromSplitGRPCRouteMatch(match) - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. kongService, _ := generateKongServiceFromBackendRefWithName( t.logger, t.storer, rules, serviceName, grpcRoute, - "grpcs", + t.getProtocolForKongService(grpcRoute), grpcBackendRefsToBackendRefs(grpcRouteRule.BackendRefs)..., ) kongService.Routes = append( @@ -144,3 +142,13 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] } return backendRefs } + +// getProtocolForKongService returns the protocol for the Kong service configuration. +// In order to get the protocol, provided route's parentRefs are searched for a Gateway that has the matching listening ports. +func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { + // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". + if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { + return "grpc" + } + return "grpcs" +} diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 014bf6b3a6..c7f89fc338 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -24,12 +24,27 @@ import ( var skippedTestsForTraditionalRoutes = []string{ // core conformance tests.HTTPRouteHeaderMatching.ShortName, + // There is an issue with KIC when processing this scenario. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6136 + tests.GRPCRouteListenerHostnameMatching.ShortName, + // tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may + // have some conflicts, skipping either one will still pass normally. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6144 + tests.GRPCExactMethodMatching.ShortName, +} + +var skippedTestsForExpressionRoutes = []string{ + // When processing this scenario, the Kong's expressions router requires `priority` + // to be specified for routes. + // We cannot provide that for routes that are part of the conformance suite. + tests.GRPCRouteListenerHostnameMatching.ShortName, } var traditionalRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteResponseHeaderModification, features.SupportHTTPRoutePathRewrite, @@ -43,6 +58,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, @@ -70,6 +86,7 @@ func TestGatewayConformance(t *testing.T) { supportedFeatures = traditionalRoutesSupportedFeatures mode = string(dpconf.RouterFlavorTraditionalCompatible) case dpconf.RouterFlavorExpressions: + skippedTests = skippedTestsForExpressionRoutes supportedFeatures = expressionRoutesSupportedFeatures mode = string(dpconf.RouterFlavorExpressions) default: @@ -86,6 +103,7 @@ func TestGatewayConformance(t *testing.T) { opts.SkipTests = skippedTests opts.ConformanceProfiles = sets.New( suite.GatewayHTTPConformanceProfileName, + suite.GatewayGRPCConformanceProfileName, ) opts.Implementation = conformancev1.Implementation{ Organization: metadata.Organization, diff --git a/test/conformance/suite_test.go b/test/conformance/suite_test.go index 51c37929ed..1e0abe24b8 100644 --- a/test/conformance/suite_test.go +++ b/test/conformance/suite_test.go @@ -82,6 +82,10 @@ func TestMain(m *testing.M) { kongBuilder = kongBuilder.WithProxyEnvVar("router_flavor", string(dpconf.RouterFlavorExpressions)) } + // The test cases for GRPCRoute in the current GatewayAPI all use the h2c protocol. + // In order to pass conformance tests, the proxy must listen http2 and http on the same port. + kongBuilder.WithProxyEnvVar("PROXY_LISTEN", `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`) + // Pin the Helm chart version. kongBuilder.WithHelmChartVersion(testenv.KongHelmChartVersion()) diff --git a/test/consts.go b/test/consts.go index 7bdb4e05fb..65d47d268e 100644 --- a/test/consts.go +++ b/test/consts.go @@ -24,7 +24,9 @@ const ( // GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool. // See: https://github.com/Kong/grpcbin GRPCBinImage = "kong/grpcbin:latest" - GRPCBinPort = 9001 + + GRPCBinPort int32 = 9000 + GRPCSBinPort int32 = 9001 // EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the // testing environment to perform its cleanup when the test suite is shutting down. diff --git a/test/integration/isolated/grpc_test.go b/test/integration/isolated/grpc_test.go index 0681cbd974..a1388ef328 100644 --- a/test/integration/isolated/grpc_test.go +++ b/test/integration/isolated/grpc_test.go @@ -22,7 +22,6 @@ import ( "google.golang.org/grpc/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" @@ -30,7 +29,6 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder" "github.com/kong/kubernetes-ingress-controller/v3/test" - "github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate" "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" @@ -43,9 +41,12 @@ func TestGRPCRouteEssentials(t *testing.T) { New("essentials"). WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). WithLabel(testlabels.Kind, testlabels.KindGRPCRoute). - WithSetup("deploy kong addon into cluster", featureSetup()). - Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTPS", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - // On purpose omit protocol annotation to test defaulting to "grpcs" that is preserved to not break users' configs. + WithSetup("deploy kong addon into cluster", featureSetup( + withKongProxyEnvVars(map[string]string{ + "PROXY_LISTEN": `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`, + }), + )). + Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTP", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t) cluster := GetClusterFromCtx(ctx) namespace := GetNamespaceForT(ctx, t) @@ -61,45 +62,12 @@ func TestGRPCRouteEssentials(t *testing.T) { assert.NoError(t, err) cleaner.Add(gwc) - t.Log("configuring secret") - const tlsRouteHostname = "tls-route.example" - tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - const tlsSecretName = "secret-test" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"), - Name: tlsSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{ - "tls.crt": tlsRouteExampleTLSCert, - "tls.key": tlsRouteExampleTLSKey, - }, - } - - t.Log("deploying secret") - secret, err = cluster.Client().CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - cleaner.Add(secret) - t.Log("deploying a new gateway") gateway, err := helpers.DeployGateway(ctx, gatewayClient, namespace, gatewayClassName, func(gw *gatewayapi.Gateway) { - // Besides default HTTP listener, add a HTTPS listener. - gw.Spec.Listeners = append( - gw.Spec.Listeners, - builder.NewListener("https"). - HTTPS(). - WithPort(ktfkong.DefaultProxyTLSServicePort). - WithHostname(testHostname). - WithTLSConfig(&gatewayapi.GatewayTLSConfig{ - CertificateRefs: []gatewayapi.SecretObjectReference{ - { - Name: gatewayapi.ObjectName(secret.Name), - }, - }, - }). - Build(), - ) + gw.Spec.Listeners = builder.NewListener("grpc"). + HTTP(). + WithPort(ktfkong.DefaultProxyHTTPPort). + IntoSlice() }) assert.NoError(t, err) cleaner.Add(gateway) @@ -168,7 +136,7 @@ func TestGRPCRouteEssentials(t *testing.T) { return ctx }). Assess("checking if GRPCRoute is linked correctly and client can connect properly to the exposed service", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - grpcAddr := GetHTTPSURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTPS, but without the scheme (https://). + grpcAddr := GetHTTPURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTP, but without the scheme (http://). namespace := GetNamespaceForT(ctx, t) gatewayClient := GetFromCtxForT[*gatewayclient.Clientset](ctx, t) grpcRoute := GetFromCtxForT[*gatewayapi.GRPCRoute](ctx, t) @@ -184,14 +152,14 @@ func TestGRPCRouteEssentials(t *testing.T) { t.Log("waiting for routes from GRPCRoute to become operational") assert.Eventually(t, func() bool { - err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", true) + err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", false) if err != nil { t.Log(err) } return err == nil }, consts.IngressWait, consts.WaitTick) - client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, true) + client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, false) assert.NoError(t, err) t.Cleanup(func() { err := closeGrpcConn() diff --git a/test/integration/isolated/ingress_test.go b/test/integration/isolated/ingress_test.go index 6b14d4b851..ef531444d8 100644 --- a/test/integration/isolated/ingress_test.go +++ b/test/integration/isolated/ingress_test.go @@ -72,23 +72,19 @@ func TestIngressGRPC(t *testing.T) { gRPC kongProtocolAnnotation = "grpc" gRPCS kongProtocolAnnotation = "grpcs" ) - const ( - gRPCBinPort int32 = 9000 - gRPCSBinPort int32 = 9001 - ) t.Log("deploying a minimal gRPC container deployment to test Ingress routes") container := generators.NewContainer("grpcbin", test.GRPCBinImage, 0) // Overwrite ports to specify gRPC over HTTP (9000) and gRPC over HTTPS (9001). - container.Ports = []corev1.ContainerPort{{ContainerPort: gRPCBinPort, Name: string(gRPC)}, {ContainerPort: gRPCSBinPort, Name: string(gRPCS)}} + container.Ports = []corev1.ContainerPort{{ContainerPort: test.GRPCBinPort, Name: string(gRPC)}, {ContainerPort: test.GRPCSBinPort, Name: string(gRPCS)}} deployment := generators.NewDeploymentForContainer(container) deployment, err = cluster.Client().AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) assert.NoError(t, err) cleaner.Add(deployment) exposeWithService := func(p kongProtocolAnnotation) *corev1.Service { - grpcBinPort := gRPCBinPort + grpcBinPort := test.GRPCBinPort if p == gRPCS { - grpcBinPort = gRPCSBinPort + grpcBinPort = test.GRPCSBinPort } kongProtocol := string(p) t.Logf("exposing deployment gRPC (%s) port %s via service", kongProtocol, deployment.Name) @@ -122,7 +118,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPCS.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCSBinPort, + Number: test.GRPCSBinPort, }, }, }, @@ -142,7 +138,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPC.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCBinPort, + Number: test.GRPCBinPort, }, }, }, diff --git a/test/internal/helpers/gatewayapi.go b/test/internal/helpers/gatewayapi.go index e0d2780758..8ea6169cf8 100644 --- a/test/internal/helpers/gatewayapi.go +++ b/test/internal/helpers/gatewayapi.go @@ -116,7 +116,7 @@ func gatewayLinkStatusMatches( switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", gerr) @@ -211,7 +211,7 @@ func verifyProgrammedConditionStatus(t *testing.T, switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err) From d7edbb63b58cd367f2742477b4cd5627582bcfaf Mon Sep 17 00:00:00 2001 From: Tao Yi Date: Thu, 6 Jun 2024 18:27:55 +0800 Subject: [PATCH 39/48] add integration tests for custom entities (#6137) * add integration tests for custom entities * add example of kong custom entity usage * move custom entity test to isolated --- examples/kong-custom-entity.yaml | 142 ++++++++++++++++++ test/consts/feature_gates.go | 2 +- .../isolated/custom_entity_test.go | 138 +++++++++++++++++ test/internal/testlabels/labels.go | 1 + 4 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 examples/kong-custom-entity.yaml create mode 100644 test/integration/isolated/custom_entity_test.go diff --git a/examples/kong-custom-entity.yaml b/examples/kong-custom-entity.yaml new file mode 100644 index 0000000000..f6542cee3c --- /dev/null +++ b/examples/kong-custom-entity.yaml @@ -0,0 +1,142 @@ +# This example demonstrates how to use KongCustomEntity custom resource to +# specify custom Kong entities. +# The example requires KIC to set `KongCustomEntity` and `FillIDs` feature gates enabled. +# The example will use `degraphql_routes` entity and `degraphql` plugin for +# demonstration. Since `degraphql` plugin can be only used with Kong gateway +# enterprise, the demo can be only used with Kong gateway enterprise installed. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hasura + hasuraService: custom + name: hasura + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: hasura + template: + metadata: + labels: + app: hasura + spec: + containers: + - image: hasura/graphql-engine:v2.38.0 + imagePullPolicy: IfNotPresent + name: hasura + env: + - name: HASURA_GRAPHQL_DATABASE_URL + value: postgres://user:password@localhost:5432/hasura_data + ## enable the console served by server + - name: HASURA_GRAPHQL_ENABLE_CONSOLE + value: "true" + ## enable debugging mode. It is recommended to disable this in production + - name: HASURA_GRAPHQL_DEV_MODE + value: "true" + ports: + - name: http + containerPort: 8080 + protocol: TCP + resources: {} + - image: postgres:15 + name: postgres + env: + - name: POSTGRES_USER + value: "user" + - name: POSTGRES_PASSWORD + value: "password" + - name: POSTGRES_DB + value: "hasura_data" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: hasura + name: hasura + namespace: default +spec: + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + selector: + app: hasura +--- +# This is the ingress that exposes the console of hasura server. +# You can access http://${PROXY_IP}/console to open hasura's console to configure data. +# See: https://hasura.io/docs/latest/getting-started/docker-simple/#step-2-connect-a-database from step 2 and over. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hasura-ingress-console + annotations: + konghq.com/strip-path: "true" +spec: + ingressClassName: kong + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hasura + port: + number: 80 +--- +# This is the ingress to expose graqhql services. +# Because we attached the `degraphql` plugin to the ingress, regular route matching is not available. +# So we cannot access the console, then we used two ingresses for console and graphQL service. +# You could use `curl -H"Host:graphql.service.example" http://${PROXY_IP}/...` to test function of degraphql plugin. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hasura-ingress-graphql + annotations: + konghq.com/strip-path: "true" + konghq.com/plugins: "degraphql-example" +spec: + ingressClassName: kong + rules: + - host: "graphql.service.example" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hasura + port: + number: 80 +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + namespace: default + name: degraphql-example +plugin: degraphql +config: + graphql_server_path: /v1/graphql +--- +# This route serves endpoint `/contacts` which extracts column `name` of all rows in `contacts` table in your `hasura_data` DB. +# You can use other query in the `query` field for fetching other data. +apiVersion: configuration.konghq.com/v1alpha1 +kind: KongCustomEntity +metadata: + namespace: default + name: degraphql-route-example +spec: + controllerName: kong + type: degraphql_routes + parentRef: + group: "configuration.konghq.com" + kind: "KongPlugin" + name: "degraphql-example" + fields: + uri: "/contacts" + query: "query{ contacts { name } }" + diff --git a/test/consts/feature_gates.go b/test/consts/feature_gates.go index 8583c7ee4a..377473c1bb 100644 --- a/test/consts/feature_gates.go +++ b/test/consts/feature_gates.go @@ -5,5 +5,5 @@ const ( // provided if none are provided by the user. This generally includes features // that are innocuous, or otherwise don't actually get triggered unless the // user takes further action. - DefaultFeatureGates = "GatewayAlpha=true,KongServiceFacade=true" + DefaultFeatureGates = "GatewayAlpha=true,KongServiceFacade=true,KongCustomEntity=true" ) diff --git a/test/integration/isolated/custom_entity_test.go b/test/integration/isolated/custom_entity_test.go new file mode 100644 index 0000000000..b898081891 --- /dev/null +++ b/test/integration/isolated/custom_entity_test.go @@ -0,0 +1,138 @@ +//go:build integration_tests + +package isolated + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "strings" + "testing" + + "github.com/kong/kubernetes-testing-framework/pkg/clusters" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" + "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" + "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" +) + +func TestCustomEntityExample(t *testing.T) { + f := features. + New("example"). + WithLabel(testlabels.Example, testlabels.ExampleTrue). + WithLabel(testlabels.Kind, testlabels.KindKongCustomEntity). + WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyIngress). + Setup(SkipIfEnterpriseNotEnabled). + Setup(SkipIfDBBacked). + WithSetup("deploy kong addon into cluster", featureSetup( + withControllerManagerOpts(helpers.ControllerManagerOptAdditionalWatchNamespace("default")), + )). + WithSetup("deploy example manifest", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + manifestPath := examplesManifestPath("kong-custom-entity.yaml") + + b, err := os.ReadFile(manifestPath) + require.NoError(t, err) + manifest := string(b) + + ingressClass := GetIngressClassFromCtx(ctx) + + t.Logf("replacing kong ingress class in yaml manifest to %s", ingressClass) + manifest = strings.ReplaceAll( + manifest, + "kubernetes.io/ingress.class: kong", + fmt.Sprintf("kubernetes.io/ingress.class: %s", ingressClass), + ) + manifest = strings.ReplaceAll( + manifest, + "ingressClassName: kong", + fmt.Sprintf("ingressClassName: %s", ingressClass), + ) + manifest = strings.ReplaceAll( + manifest, + "controllerName: kong", + fmt.Sprintf("controllerName: %s", ingressClass), + ) + + t.Logf("applying yaml manifest %s", manifestPath) + cluster := GetClusterFromCtx(ctx) + require.NoError(t, clusters.ApplyManifestByYAML(ctx, cluster, manifest)) + + t.Cleanup(func() { + t.Logf("deleting yaml manifest %s", manifestPath) + assert.NoError(t, clusters.DeleteManifestByYAML(ctx, cluster, manifest)) + }) + + t.Log("waiting for hasura deployment to be ready") + helpers.WaitForDeploymentRollout(ctx, t, cluster, "default", "hasura") + return ctx + }). + Assess("degraphql plugin works as expected", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + proxyURL := GetHTTPURLFromCtx(ctx) + t.Log("Waiting for graphQL service to be available") + helpers.EventuallyGETPath(t, proxyURL, proxyURL.Host, "/healthz", http.StatusOK, "OK", nil, consts.IngressWait, consts.WaitTick) + + t.Log("injecting data for graphQL service") + injectDataURL := proxyURL.String() + "/v2/query" + runSQLCreateTableBody := `{ + "type": "run_sql", + "args": { + "sql": "CREATE TABLE contacts(id serial NOT NULL, name text NOT NULL, phone_number text NOT NULL, PRIMARY KEY(id));" + } + }` + req, err := http.NewRequest(http.MethodPost, injectDataURL, bytes.NewReader([]byte(runSQLCreateTableBody))) + require.NoError(t, err) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-Hasura-Role", "admin") + resp, err := helpers.DefaultHTTPClient().Do(req) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + runSQLInsertRowBody := `{ + "type": "run_sql", + "args": { + "sql": "INSERT INTO contacts (name, phone_number) VALUES ('Alice','0123456789');" + } + }` + req, err = http.NewRequest(http.MethodPost, injectDataURL, bytes.NewReader([]byte(runSQLInsertRowBody))) + require.NoError(t, err) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-Hasura-Role", "admin") + resp, err = helpers.DefaultHTTPClient().Do(req) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + setMetadataURL := proxyURL.String() + "/v1/metadata" + trackTableBody := `{ + "type": "pg_track_table", + "args": { + "schema": "public", + "name": "contacts" + } + }` + req, err = http.NewRequest(http.MethodPost, setMetadataURL, bytes.NewReader([]byte(trackTableBody))) + require.NoError(t, err) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-Hasura-Role", "admin") + resp, err = helpers.DefaultHTTPClient().Do(req) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + t.Log("verifying degraphQL plugin and degraphql_routes entity works") + // The ingress providing graphQL service has a different host, so we need to set the `Host` header. + helpers.EventuallyGETPath(t, proxyURL, "graphql.service.example", "/contacts", http.StatusOK, `"name":"Alice"`, map[string]string{"Host": "graphql.service.example"}, consts.IngressWait, consts.WaitTick) + + return ctx + }). + Teardown(featureTeardown()) + + tenv.Test(t, f.Feature()) +} diff --git a/test/internal/testlabels/labels.go b/test/internal/testlabels/labels.go index 0f70a7c5bf..3c92c308dc 100644 --- a/test/internal/testlabels/labels.go +++ b/test/internal/testlabels/labels.go @@ -13,6 +13,7 @@ const ( KindKongTCPIngress = "TCPIngress" KindKongUpstreamPolicy = "KongUpstreamPolicy" KindKongLicense = "KongLicense" + KindKongCustomEntity = "KongCustomEntity" ) const ( From 1a54b96a2a89a0bb4be989e4952bd23533f93313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Thu, 6 Jun 2024 16:48:36 +0200 Subject: [PATCH 40/48] chore: add skaffold config for multi gw enterprise (#6145) --- config/base/kong-ingress-dbless.yaml | 3 +++ .../manager_dev_patch/kustomization.yaml | 5 +++++ .../manager_dev_patch}/manager.yaml | 9 ++++++--- config/image/enterprise/kustomization.yaml | 4 ++-- config/image/oss/kustomization.yaml | 4 ++-- config/variants/multi-gw/dev/kustomization.yaml | 4 +--- renovate.json | 14 ++++++++++++++ skaffold.yaml | 16 ++++++++++++++++ .../all-in-one-dbless-k4k8s-enterprise.yaml | 5 ++++- .../all-in-one-dbless-konnect-enterprise.yaml | 5 ++++- .../e2e/manifests/all-in-one-dbless-konnect.yaml | 3 +++ test/e2e/manifests/all-in-one-dbless.yaml | 3 +++ .../all-in-one-postgres-enterprise.yaml | 11 +++++++---- .../all-in-one-postgres-multiple-gateways.yaml | 3 +++ test/e2e/manifests/all-in-one-postgres.yaml | 3 +++ 15 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 config/components/manager_dev_patch/kustomization.yaml rename config/{variants/multi-gw/dev => components/manager_dev_patch}/manager.yaml (61%) diff --git a/config/base/kong-ingress-dbless.yaml b/config/base/kong-ingress-dbless.yaml index eb3debd8a4..0869770b61 100644 --- a/config/base/kong-ingress-dbless.yaml +++ b/config/base/kong-ingress-dbless.yaml @@ -134,6 +134,9 @@ spec: - name: cmetrics containerPort: 10255 protocol: TCP + - name: diagnostics + containerPort: 10256 + protocol: TCP livenessProbe: httpGet: path: /healthz diff --git a/config/components/manager_dev_patch/kustomization.yaml b/config/components/manager_dev_patch/kustomization.yaml new file mode 100644 index 0000000000..8ecb7b0a2f --- /dev/null +++ b/config/components/manager_dev_patch/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: +- path: manager.yaml diff --git a/config/variants/multi-gw/dev/manager.yaml b/config/components/manager_dev_patch/manager.yaml similarity index 61% rename from config/variants/multi-gw/dev/manager.yaml rename to config/components/manager_dev_patch/manager.yaml index 5edfd5ccbe..0d96f3ac51 100644 --- a/config/variants/multi-gw/dev/manager.yaml +++ b/config/components/manager_dev_patch/manager.yaml @@ -16,10 +16,13 @@ spec: spec: containers: - name: ingress-controller - args: - - --feature-gates=GatewayAlpha=true,KongServiceFacade=true - - --anonymous-reports=false env: + - name: CONTROLLER_DUMP_CONFIG + value: "true" - name: CONTROLLER_LOG_LEVEL value: debug + - name: CONTROLLER_FEATURE_GATES + value: GatewayAlpha=true,KongServiceFacade=true,FallbackConfiguration=true + - name: CONTROLLER_ANONYMOUS_REPORTS + value: "false" image: kic-placeholder:placeholder diff --git a/config/image/enterprise/kustomization.yaml b/config/image/enterprise/kustomization.yaml index b13d10ff82..0db707f835 100644 --- a/config/image/enterprise/kustomization.yaml +++ b/config/image/enterprise/kustomization.yaml @@ -6,7 +6,7 @@ kind: Component images: - name: kong newName: kong/kong-gateway - newTag: '3.5' + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway - name: kong-placeholder newName: kong/kong-gateway - newTag: '3.5' + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway diff --git a/config/image/oss/kustomization.yaml b/config/image/oss/kustomization.yaml index 5316d15907..e2a0681e08 100644 --- a/config/image/oss/kustomization.yaml +++ b/config/image/oss/kustomization.yaml @@ -4,7 +4,7 @@ kind: Component images: - name: kong-placeholder newName: kong - newTag: '3.7' + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong - name: kic-placeholder newName: kong/kubernetes-ingress-controller - newTag: '3.1' + newTag: '3.1' # renovate: datasource=docker versioning=docker depName=kong/kubernetes-ingress-controller diff --git a/config/variants/multi-gw/dev/kustomization.yaml b/config/variants/multi-gw/dev/kustomization.yaml index 67cd7c3ce3..b564141ce6 100644 --- a/config/variants/multi-gw/dev/kustomization.yaml +++ b/config/variants/multi-gw/dev/kustomization.yaml @@ -8,6 +8,4 @@ resources: components: - ../../../components/manager_dev_webhook - -patches: -- path: manager.yaml +- ../../../components/manager_dev_patch diff --git a/renovate.json b/renovate.json index 33fefcca13..539aa24f0d 100644 --- a/renovate.json +++ b/renovate.json @@ -61,6 +61,20 @@ "addLabels": [ "renovate/auto-regenerate" ] + }, + { + "description": "Match versions in config/image/oss and config/image/enterprise kustomize files that are properly annotated with `# renovate: datasource={} versioning={} depName={}`.", + "customType": "regex", + "fileMatch": [ + "^config/image/enterprise/.*\\.yaml$", + "^config/image/oss/.*\\.yaml$" + ], + "matchStrings": [ + "'(?.+)' # renovate: datasource=(?.*) versioning=(?.*) depName=(?.+)" + ], + "addLabels": [ + "renovate/auto-regenerate" + ] } ] } diff --git a/skaffold.yaml b/skaffold.yaml index 942f46d2d7..f5bceb6681 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -62,6 +62,22 @@ profiles: COMMIT: ${{ .COMMIT }} REPO_INFO: ${{ .REPO_INFO }} GOCACHE: "{{ .GOCACHE }}" +- name: multi_gw_enterprise + manifests: + kustomize: + paths: + - config/variants/multi-gw/enterprise + build: + artifacts: + - image: kic-placeholder + docker: + dockerfile: Dockerfile + target: distroless + buildArgs: + TAG: ${{ .TAG }} + COMMIT: ${{ .COMMIT }} + REPO_INFO: ${{ .REPO_INFO }} + GOCACHE: "{{ .GOCACHE }}" - name: multi_gw_postgres manifests: kustomize: diff --git a/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml b/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml index 24b46aba63..8c47858a1a 100644 --- a/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-dbless-k4k8s-enterprise.yaml @@ -3788,6 +3788,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: @@ -3874,7 +3877,7 @@ spec: value: /dev/stderr - name: KONG_ROUTER_FLAVOR value: expressions - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 lifecycle: preStop: exec: diff --git a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml index f40da8654a..84a68148ef 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect-enterprise.yaml @@ -3803,6 +3803,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: @@ -3884,7 +3887,7 @@ spec: value: /dev/stderr - name: KONG_ROUTER_FLAVOR value: expressions - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 lifecycle: preStop: exec: diff --git a/test/e2e/manifests/all-in-one-dbless-konnect.yaml b/test/e2e/manifests/all-in-one-dbless-konnect.yaml index f0aa713bfe..f920fe2e76 100644 --- a/test/e2e/manifests/all-in-one-dbless-konnect.yaml +++ b/test/e2e/manifests/all-in-one-dbless-konnect.yaml @@ -3805,6 +3805,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: diff --git a/test/e2e/manifests/all-in-one-dbless.yaml b/test/e2e/manifests/all-in-one-dbless.yaml index c167555156..3a2d1caddf 100644 --- a/test/e2e/manifests/all-in-one-dbless.yaml +++ b/test/e2e/manifests/all-in-one-dbless.yaml @@ -3790,6 +3790,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: diff --git a/test/e2e/manifests/all-in-one-postgres-enterprise.yaml b/test/e2e/manifests/all-in-one-postgres-enterprise.yaml index ca7ab78a50..c46b036614 100644 --- a/test/e2e/manifests/all-in-one-postgres-enterprise.yaml +++ b/test/e2e/manifests/all-in-one-postgres-enterprise.yaml @@ -3823,7 +3823,7 @@ spec: value: /dev/stderr - name: KONG_PROXY_ERROR_LOG value: /dev/stderr - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 lifecycle: preStop: exec: @@ -3910,6 +3910,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: @@ -3942,7 +3945,7 @@ spec: value: postgres - name: KONG_PG_PASSWORD value: kong - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 name: wait-for-migrations serviceAccountName: kong-serviceaccount volumes: @@ -4041,7 +4044,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 name: kong-migrations imagePullSecrets: - name: kong-enterprise-edition-docker @@ -4056,7 +4059,7 @@ spec: value: postgres - name: KONG_PG_PORT value: "5432" - image: kong/kong-gateway:3.5 + image: kong/kong-gateway:3.7 name: wait-for-postgres restartPolicy: OnFailure --- diff --git a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml index 8fdc8e9bea..eec7bff0c2 100644 --- a/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml +++ b/test/e2e/manifests/all-in-one-postgres-multiple-gateways.yaml @@ -3804,6 +3804,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: diff --git a/test/e2e/manifests/all-in-one-postgres.yaml b/test/e2e/manifests/all-in-one-postgres.yaml index 5a8a3e4985..3e651cdb22 100644 --- a/test/e2e/manifests/all-in-one-postgres.yaml +++ b/test/e2e/manifests/all-in-one-postgres.yaml @@ -3854,6 +3854,9 @@ spec: - containerPort: 10255 name: cmetrics protocol: TCP + - containerPort: 10256 + name: diagnostics + protocol: TCP readinessProbe: failureThreshold: 3 httpGet: From edd44190be4c177f03ec72cd856e959c9757d15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Thu, 6 Jun 2024 16:50:16 +0200 Subject: [PATCH 41/48] tests: add UT for deckgen.fillPlugin (#6146) --- internal/dataplane/deckgen/generate_test.go | 553 +++++++++++++++++++- 1 file changed, 546 insertions(+), 7 deletions(-) diff --git a/internal/dataplane/deckgen/generate_test.go b/internal/dataplane/deckgen/generate_test.go index 3f75522bd5..3fe63f74d8 100644 --- a/internal/dataplane/deckgen/generate_test.go +++ b/internal/dataplane/deckgen/generate_test.go @@ -1,4 +1,4 @@ -package deckgen_test +package deckgen import ( "context" @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" - "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/deckgen" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/kongstate" "github.com/kong/kubernetes-ingress-controller/v3/internal/versions" ) @@ -19,13 +18,13 @@ import ( func TestToDeckContent(t *testing.T) { testCases := []struct { name string - params deckgen.GenerateDeckContentParams + params GenerateDeckContentParams input *kongstate.KongState expected *file.Content }{ { name: "empty", - params: deckgen.GenerateDeckContentParams{}, + params: GenerateDeckContentParams{}, input: &kongstate.KongState{}, expected: &file.Content{ FormatVersion: versions.DeckFileFormatVersion, @@ -33,7 +32,7 @@ func TestToDeckContent(t *testing.T) { }, { name: "empty, generate stub entity", - params: deckgen.GenerateDeckContentParams{ + params: GenerateDeckContentParams{ AppendStubEntityWhenConfigEmpty: true, }, input: &kongstate.KongState{}, @@ -42,7 +41,7 @@ func TestToDeckContent(t *testing.T) { Upstreams: []file.FUpstream{ { Upstream: kong.Upstream{ - Name: lo.ToPtr(deckgen.StubUpstreamName), + Name: lo.ToPtr(StubUpstreamName), }, }, }, @@ -52,8 +51,548 @@ func TestToDeckContent(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := deckgen.ToDeckContent(context.Background(), zapr.NewLogger(zap.NewNop()), tc.input, tc.params) + result := ToDeckContent(context.Background(), zapr.NewLogger(zap.NewNop()), tc.input, tc.params) require.Equal(t, tc.expected, result) }) } } + +func TestFillPlugin(t *testing.T) { + testCases := []struct { + name string + plugin *file.FPlugin + schemas PluginSchemaStore + expected *file.FPlugin + expectedError error + }{ + { + name: "Required field provided for plugin", + plugin: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("plugin"), + Config: kong.Configuration{ + "endpoint": "https://example.com", + }, + }, + }, + schemas: &mockPluginSchemaStore{ + map[string]interface{}{ + "fields": []interface{}{ + map[string]interface{}{ + "protocols": map[string]interface{}{ + "elements": map[string]interface{}{ + "type": "string", + "one_of": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + }, + "description": "A set of strings representing HTTP protocols.", + "type": "set", + "default": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + "required": true, + }, + }, + map[string]interface{}{ + "config": map[string]interface{}{ + "type": "record", + "fields": []interface{}{ + map[string]interface{}{ + "endpoint": map[string]interface{}{ + "type": "string", + "required": true, + "description": "A string representing a URL, such as https://example.com/path/to/resource?q=search.", + "referenceable": true, + }, + }, + }, + }, + }, + }, + }, + }, + expected: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("plugin"), + Protocols: []*string{ + lo.ToPtr("grpc"), + lo.ToPtr("grpcs"), + lo.ToPtr("http"), + lo.ToPtr("https"), + }, + Enabled: lo.ToPtr(true), + Config: kong.Configuration{ + "endpoint": "https://example.com", + }, + }, + }, + }, + { + name: "Required field not provided for plugin gets filled in with nil", + plugin: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("plugin"), + }, + }, + schemas: &mockPluginSchemaStore{ + map[string]interface{}{ + "fields": []interface{}{ + map[string]interface{}{ + "protocols": map[string]interface{}{ + "elements": map[string]interface{}{ + "type": "string", + "one_of": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + }, + "description": "A set of strings representing HTTP protocols.", + "type": "set", + "default": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + "required": true, + }, + }, + map[string]interface{}{ + "config": map[string]interface{}{ + "type": "record", + "fields": []interface{}{ + map[string]interface{}{ + "endpoint": map[string]interface{}{ + "type": "string", + "required": true, + "description": "A string representing a URL, such as https://example.com/path/to/resource?q=search.", + "referenceable": true, + }, + }, + }, + }, + }, + }, + }, + }, + expected: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("plugin"), + Protocols: []*string{ + lo.ToPtr("grpc"), + lo.ToPtr("grpcs"), + lo.ToPtr("http"), + lo.ToPtr("https"), + }, + Enabled: lo.ToPtr(true), + Config: kong.Configuration{ + "endpoint": nil, + }, + }, + }, + }, + { + // NOTE: This would fail for go-kong v0.52.0 and older. + name: "OpenTelemetry plugin for Gateway 3.7.x", + plugin: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("opentelemetry"), + }, + }, + schemas: &mockPluginSchemaStore{ + schema: map[string]interface{}{ + "fields": []interface{}{ + map[string]interface{}{ + "protocols": map[string]interface{}{ + "elements": map[string]interface{}{ + "type": "string", + "one_of": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + }, + "description": "A set of strings representing HTTP protocols.", + "type": "set", + "default": []interface{}{ + "grpc", + "grpcs", + "http", + "https", + }, + "required": true, + }, + }, + map[string]interface{}{ + "config": map[string]interface{}{ + "type": "record", + "fields": []interface{}{ + map[string]interface{}{ + "endpoint": map[string]interface{}{ + "type": "string", + "required": true, + "description": "A string representing a URL, such as https://example.com/path/to/resource?q=search.", + "referenceable": true, + }, + }, + map[string]interface{}{ + "headers": map[string]interface{}{ + "description": "The custom headers to be added in the HTTP request sent to the OTLP server. This setting is useful for adding the authentication headers (token) for the APM backend.", + "type": "map", + "values": map[string]interface{}{ + "type": "string", + "referenceable": true, + }, + "keys": map[string]interface{}{ + "type": "string", + "description": "A string representing an HTTP header name.", + }, + }, + }, + map[string]interface{}{ + "resource_attributes": map[string]interface{}{ + "type": "map", + "keys": map[string]interface{}{ + "type": "string", + "required": true, + }, + "values": map[string]interface{}{ + "type": "string", + "required": true, + }, + "description": "Attributes to add to the OpenTelemetry resource object, following the spec for Semantic Attributes. \nThe following attributes are automatically added:\n- \"service.name\": The name of the service (default:'kong').\n-'service.version': The version of Kong Gateway.\n-'service.instance.id': The node ID of Kong Gateway.\n\nYou can use this property to override default attribute values. For example, to override the default for'service.name', you can specify'{ \"service.name\": \"my-service\" }'.", + }, + }, + map[string]interface{}{ + "queue": map[string]interface{}{ + "type": "record", + "fields": []interface{}{ + map[string]interface{}{ + "max_batch_size": map[string]interface{}{ + "type": "integer", + "between": []interface{}{ + 1, + 1000000, + }, + "default": 1, + "description": "Maximum number of entries that can be processed at a time.", + }, + }, + map[string]interface{}{ + "max_coalescing_delay": map[string]interface{}{ + "type": "number", + "between": []interface{}{ + 0, + 3600, + }, + "default": 1, + "description": "Maximum number of (fractional) seconds to elapse after the first entry was queued before the queue starts calling the handler.", + }, + }, + map[string]interface{}{ + "max_entries": map[string]interface{}{ + "type": "integer", + "between": []interface{}{ + 1, + 1000000, + }, + "default": 10000, + "description": "Maximum number of entries that can be waiting on the queue.", + }, + }, + map[string]interface{}{ + "max_bytes": map[string]interface{}{ + "type": "integer", + "description": "Maximum number of bytes that can be waiting on a queue, requires string content.", + }, + }, + map[string]interface{}{ + "max_retry_time": map[string]interface{}{ + "type": "number", + "default": 60, + "description": "Time in seconds before the queue gives up calling a failed handler for a batch.", + }, + }, + map[string]interface{}{ + "initial_retry_delay": map[string]interface{}{ + "type": "number", + "between": []interface{}{ + 0.001, + 1000000, + }, + "default": 0.01, + "description": "Time in seconds before the initial retry is made for a failing batch.", + }, + }, + map[string]interface{}{ + "max_retry_delay": map[string]interface{}{ + "type": "number", + "between": []interface{}{ + 0.001, + 1000000, + }, + "default": 60, + "description": "Maximum time in seconds between retries, caps exponential backoff.", + }, + }, + }, + "default": map[string]interface{}{ + "max_batch_size": 200, + }, + "required": true, + }, + }, + map[string]interface{}{ + "batch_span_count": map[string]interface{}{ + "description": "The number of spans to be sent in a single batch.", + "type": "integer", + "deprecation": map[string]interface{}{ + "old_default": 200, + "removal_in_version": "4.0", + "message": "opentelemetry: config.batch_span_count is deprecated, please use config.queue.max_batch_size instead", + }, + }, + }, + map[string]interface{}{ + "batch_flush_delay": map[string]interface{}{ + "description": "The delay, in seconds, between two consecutive batches.", + "type": "integer", + "deprecation": map[string]interface{}{ + "old_default": 3, + "removal_in_version": "4.0", + "message": "opentelemetry: config.batch_flush_delay is deprecated, please use config.queue.max_coalescing_delay instead", + }, + }, + }, + map[string]interface{}{ + "connect_timeout": map[string]interface{}{ + "type": "integer", + "description": "An integer representing a timeout in milliseconds. Must be between 0 and 2^31-2.", + "default": 1000, + "between": []interface{}{ + 0, + 2147483646, + }, + }, + }, + map[string]interface{}{ + "send_timeout": map[string]interface{}{ + "type": "integer", + "description": "An integer representing a timeout in milliseconds. Must be between 0 and 2^31-2.", + "default": 5000, + "between": []interface{}{ + 0, + 2147483646, + }, + }, + }, + map[string]interface{}{ + "read_timeout": map[string]interface{}{ + "type": "integer", + "description": "An integer representing a timeout in milliseconds. Must be between 0 and 2^31-2.", + "default": 5000, + "between": []interface{}{ + 0, + 2147483646, + }, + }, + }, + map[string]interface{}{ + "http_response_header_for_traceid": map[string]interface{}{ + "description": "Specifies a custom header for the'trace_id'. If set, the plugin sets the corresponding header in the response.", + "type": "string", + }, + }, + map[string]interface{}{ + "header_type": map[string]interface{}{ + "deprecation": map[string]interface{}{ + "old_default": "preserve", + "removal_in_version": "4.0", + "message": "opentelemetry: config.header_type is deprecated, please use config.propagation options instead", + }, + "one_of": []interface{}{ + "preserve", + "ignore", + "b3", + "b3-single", + "w3c", + "jaeger", + "ot", + "aws", + "gcp", + "datadog", + }, + "type": "string", + "description": "All HTTP requests going through the plugin are tagged with a tracing HTTP request. This property codifies what kind of tracing header the plugin expects on incoming requests.", + "default": "preserve", + "required": false, + }, + }, + map[string]interface{}{ + "sampling_rate": map[string]interface{}{ + "between": []interface{}{ + 0, + 1, + }, + "description": "Tracing sampling rate for configuring the probability-based sampler. When set, this value supersedes the global'tracing_sampling_rate' setting from kong.conf.", + "type": "number", + "required": false, + }, + }, + map[string]interface{}{ + "propagation": map[string]interface{}{ + "type": "record", + "fields": []interface{}{ + map[string]interface{}{ + "extract": map[string]interface{}{ + "description": "Header formats used to extract tracing context from incoming requests. If multiple values are specified, the first one found will be used for extraction. If left empty, Kong will not extract any tracing context information from incoming requests and generate a trace with no parent and a new trace ID.", + "type": "array", + "elements": map[string]interface{}{ + "type": "string", + "one_of": []interface{}{ + "ot", + "w3c", + "datadog", + "b3", + "gcp", + "jaeger", + "aws", + }, + }, + }, + }, + map[string]interface{}{ + "clear": map[string]interface{}{ + "description": "Header names to clear after context extraction. This allows to extract the context from a certain header and then remove it from the request, useful when extraction and injection are performed on different header formats and the original header should not be sent to the upstream. If left empty, no headers are cleared.", + "type": "array", + "elements": map[string]interface{}{ + "type": "string", + }, + }, + }, + map[string]interface{}{ + "inject": map[string]interface{}{ + "description": "Header formats used to inject tracing context. The value 'preserve' will use the same header format as the incoming request. If multiple values are specified, all of them will be used during injection. If left empty, Kong will not inject any tracing context information in outgoing requests.", + "type": "array", + "elements": map[string]interface{}{ + "type": "string", + "one_of": []interface{}{ + "preserve", + "ot", + "w3c", + "datadog", + "b3", + "gcp", + "b3-single", + "jaeger", + "aws", + }, + }, + }, + }, + map[string]interface{}{ + "default_format": map[string]interface{}{ + "description": "The default header format to use when extractors did not match any format in the incoming headers and'inject' is configured with the value:'preserve'. This can happen when no tracing header was found in the request, or the incoming tracing header formats were not included in'extract'.", + "one_of": []interface{}{ + "ot", + "w3c", + "datadog", + "b3", + "gcp", + "b3-single", + "jaeger", + "aws", + }, + "type": "string", + "required": true, + }, + }, + }, + "default": map[string]interface{}{ + "default_format": "w3c", + }, + "required": true, + }, + }, + }, + "required": true, + }, + }, + }, + "entity_checks": []interface{}{}, + }, + }, + expected: &file.FPlugin{ + Plugin: kong.Plugin{ + Name: lo.ToPtr("opentelemetry"), + Protocols: []*string{ + lo.ToPtr("grpc"), + lo.ToPtr("grpcs"), + lo.ToPtr("http"), + lo.ToPtr("https"), + }, + Enabled: lo.ToPtr(true), + Config: kong.Configuration{ + "endpoint": nil, + "batch_flush_delay": nil, + "batch_span_count": nil, + "connect_timeout": float64(1000), + "header_type": "preserve", + "headers": nil, + "http_response_header_for_traceid": nil, + "propagation": map[string]interface{}{ + "clear": nil, + "default_format": "w3c", + "extract": nil, + "inject": nil, + }, + "queue": map[string]interface{}{ + "initial_retry_delay": float64(0.01), + "max_batch_size": float64(200), + "max_bytes": nil, + "max_coalescing_delay": float64(1), + "max_entries": float64(10000), + "max_retry_delay": float64(60), + "max_retry_time": float64(60), + }, + "read_timeout": float64(5000), + "resource_attributes": nil, + "sampling_rate": nil, + "send_timeout": float64(5000), + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + plugin := tc.plugin.DeepCopy() + err := fillPlugin(context.Background(), plugin, tc.schemas) + if tc.expectedError != nil { + require.EqualError(t, err, tc.expectedError.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, plugin) + } + }) + } +} + +type mockPluginSchemaStore struct { + schema map[string]interface{} +} + +func (m *mockPluginSchemaStore) Schema(_ context.Context, _ string) (map[string]interface{}, error) { + return m.schema, nil +} From 8fa070515a975e4c2b93ef414b7c8412d732cf68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:23:11 +0200 Subject: [PATCH 42/48] chore(deps): bump sigs.k8s.io/controller-runtime from 0.18.3 to 0.18.4 (#6150) Bumps [sigs.k8s.io/controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) from 0.18.3 to 0.18.4. - [Release notes](https://github.com/kubernetes-sigs/controller-runtime/releases) - [Changelog](https://github.com/kubernetes-sigs/controller-runtime/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes-sigs/controller-runtime/compare/v0.18.3...v0.18.4) --- updated-dependencies: - dependency-name: sigs.k8s.io/controller-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 482bd32269..6aceff118c 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( k8s.io/apimachinery v0.30.1 k8s.io/client-go v0.30.1 k8s.io/component-base v0.30.1 - sigs.k8s.io/controller-runtime v0.18.3 + sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/e2e-framework v0.4.0 sigs.k8s.io/gateway-api v1.1.0 sigs.k8s.io/kustomize/api v0.17.2 diff --git a/go.sum b/go.sum index ea4ca18773..e8dd1767f8 100644 --- a/go.sum +++ b/go.sum @@ -657,8 +657,8 @@ k8s.io/kubectl v0.30.1 h1:sHFIRI3oP0FFZmBAVEE8ErjnTyXDPkBcvO88mH9RjuY= k8s.io/kubectl v0.30.1/go.mod h1:7j+L0Cc38RYEcx+WH3y44jRBe1Q1jxdGPKkX0h4iDq0= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.18.3 h1:B5Wmmo8WMWK7izei+2LlXLVDGzMwAHBNLX68lwtlSR4= -sigs.k8s.io/controller-runtime v0.18.3/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY= sigs.k8s.io/e2e-framework v0.4.0/go.mod h1:JilFQPF1OL1728ABhMlf9huse7h+uBJDXl9YeTs49A8= sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= From 36ad612fdf695b14b8492c7d105011db9fb7f656 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:33:26 +0200 Subject: [PATCH 43/48] chore(deps): bump github.com/docker/docker (#6149) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 26.1.3+incompatible to 26.1.4+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v26.1.3...v26.1.4) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6aceff118c..6f49ffd0c8 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.5.0 // indirect - github.com/docker/docker v26.1.3+incompatible + github.com/docker/docker v26.1.4+incompatible github.com/docker/go-connections v0.5.0 github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect diff --git a/go.sum b/go.sum index e8dd1767f8..7b5bdd94a8 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= -github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= +github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= From 7dc67be4e6ec24d2652fef091bf8cc42a0d5b616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 6 Jun 2024 18:59:20 +0200 Subject: [PATCH 44/48] chore(tests): convert Translator golden tests into KongClient ones (#6147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts the Translator golden tests into KongClient ones. This will allow us to cover the whole KongClient.Update flow, making these tests more reliable as we will store the content of the config requests that got to the Admin API instead of the artificially marshaled file.Content that we were getting before by manually calling deckgen.ToDeckContent in the test. Co-authored-by: Patryk Małek --- Makefile | 2 +- ...den_test.go => kong_client_golden_test.go} | 158 ++++++++++++------ .../default_golden.yaml | 16 +- .../golden/consumer-group-example-ee/in.yaml | 0 .../grpcroute-example/default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../testdata/golden/grpcroute-example/in.yaml | 0 .../default_golden.yaml | 0 .../host-header-annotation-httproute/in.yaml | 0 .../httproute-example/default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../testdata/golden/httproute-example/in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../httproute-url-rewrite-path-prefix/in.yaml | 0 .../ingress-v1-empty-path/default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../golden/ingress-v1-empty-path/in.yaml | 0 .../default_golden.yaml | 0 .../in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../ingress-v1-ports-defined-by-name/in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../ingress-v1-regex-prefixed-path/in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../golden/ingress-v1-rule-with-tls/in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../ingress-v1-with-acme-like-path/in.yaml | 0 .../default_golden.yaml | 0 .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../ingress-v1-with-default-backend/in.yaml | 0 .../kong-service-facade/default_golden.yaml | 3 + .../expression-routes-on_golden.yaml | 0 .../expression-routes-on_settings.yaml | 0 .../feature-flag-on_golden.yaml | 0 .../feature-flag-on_settings.yaml | 0 .../golden/kong-service-facade/in.yaml | 0 .../default_golden.yaml | 0 .../golden/kongplugin-instance-name/in.yaml | 0 .../default_golden.yaml | 0 .../kongupstreampolicy-httproute/in.yaml | 0 .../default_golden.yaml | 0 .../golden/kongupstreampolicy-ingress/in.yaml | 0 .../default_golden.yaml | 0 .../same-name-services-single-backend/in.yaml | 0 .../kong-service-facade/default_golden.yaml | 1 - ... kong_client_golden_tests_outputs_test.go} | 65 ++++--- test/mocks/admin_api_handler.go | 26 ++- 73 files changed, 172 insertions(+), 99 deletions(-) rename internal/dataplane/{translator/golden_test.go => kong_client_golden_test.go} (64%) rename internal/dataplane/{translator => }/testdata/golden/consumer-group-example-ee/default_golden.yaml (71%) rename internal/dataplane/{translator => }/testdata/golden/consumer-group-example-ee/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/grpcroute-example/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/grpcroute-example/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/grpcroute-example/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/grpcroute-example/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/host-header-annotation-httproute/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/host-header-annotation-httproute/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-example/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-example/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-example/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-example/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-url-rewrite-path-prefix/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/httproute-url-rewrite-path-prefix/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-empty-path/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-empty-path/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-empty-path/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-empty-path/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ingress-class-annotation/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ingress-class-annotation/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-multiple-ports-for-one-service/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-multiple-ports-for-one-service/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ports-defined-by-name/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-ports-defined-by-name/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefix-exact-rule/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefix-exact-rule/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefixed-path/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-regex-prefixed-path/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-rule-with-tls/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-rule-with-tls/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-acme-like-path/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-acme-like-path/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-default-backend/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/ingress-v1-with-default-backend/in.yaml (100%) create mode 100644 internal/dataplane/testdata/golden/kong-service-facade/default_golden.yaml rename internal/dataplane/{translator => }/testdata/golden/kong-service-facade/expression-routes-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kong-service-facade/expression-routes-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kong-service-facade/feature-flag-on_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kong-service-facade/feature-flag-on_settings.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kong-service-facade/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongplugin-instance-name/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongplugin-instance-name/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongupstreampolicy-httproute/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongupstreampolicy-httproute/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongupstreampolicy-ingress/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/kongupstreampolicy-ingress/in.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/same-name-services-single-backend/default_golden.yaml (100%) rename internal/dataplane/{translator => }/testdata/golden/same-name-services-single-backend/in.yaml (100%) delete mode 100644 internal/dataplane/translator/testdata/golden/kong-service-facade/default_golden.yaml rename test/kongintegration/{translator_golden_tests_outputs_test.go => kong_client_golden_tests_outputs_test.go} (71%) diff --git a/Makefile b/Makefile index 926ee7b1f5..c4f391fbf4 100644 --- a/Makefile +++ b/Makefile @@ -423,7 +423,7 @@ test.unit.pretty: .PHONY: test.golden.update test.golden.update: - @go test -v -run TestTranslator_GoldenTests ./internal/dataplane/translator -update + @go test -v -run TestKongClient_GoldenTests ./internal/dataplane -update .PHONY: use-setup-envtest diff --git a/internal/dataplane/translator/golden_test.go b/internal/dataplane/kong_client_golden_test.go similarity index 64% rename from internal/dataplane/translator/golden_test.go rename to internal/dataplane/kong_client_golden_test.go index 79977fb16a..877a3acaaa 100644 --- a/internal/dataplane/translator/golden_test.go +++ b/internal/dataplane/kong_client_golden_test.go @@ -1,15 +1,17 @@ -package translator_test +package dataplane import ( "bytes" "context" "flag" "fmt" + "net/http/httptest" "os" "path/filepath" "reflect" "strings" "testing" + "time" "github.com/go-logr/zapr" "github.com/kong/go-kong/kong" @@ -19,17 +21,23 @@ import ( "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" - "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/deckgen" + "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" + dpconf "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/config" + "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/configfetcher" + "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback" + "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/sendconfig" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/translator" + "github.com/kong/kubernetes-ingress-controller/v3/internal/diagnostics" "github.com/kong/kubernetes-ingress-controller/v3/internal/manager/featuregates" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" + "github.com/kong/kubernetes-ingress-controller/v3/test/mocks" ) var ( - // updateGolden tells whether to update golden files using the current output of the translator. + // updateGolden tells whether to update golden files using the current config received by the Admin API. updateGolden = flag.Bool("update", false, "update golden files") - // defaultFeatureFlags is the default set of feature flags to use in tests. Can be overridden in a test case. + // defaultFeatureFlags is the default set of Translator feature flags to use in tests. Can be overridden in a test case. defaultFeatureFlags = func() translator.FeatureFlags { defaults := featuregates.GetFeatureGatesDefaults() return translator.FeatureFlags{ @@ -50,13 +58,7 @@ const ( settingsFileSuffix = "_settings.yaml" ) -type fakeSchemaServiceProvier struct{} - -func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService { - return translator.UnavailableSchemaService{} -} - -// TestTranslator_GoldenTests runs the golden tests for the translator. +// TestKongClient_GoldenTests runs the golden tests for the KongClient. // // Command to update the golden files: // $ make test.golden.update @@ -64,12 +66,13 @@ func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService // Data for the test cases is stored in the "./testdata/golden" directory. Test cases are grouped into subdirectories // based on the Kubernetes input that they run against so that each of the subdirectories has: // - an input file that represents the input with Kubernetes objects to be loaded into the store: "in.yaml", -// - a set of "_settings.yaml" files that define the translator configuration for a given test case, -// - a set of expected golden "_golden.yaml" files (in Deck format) where each file represents an +// - a set of "_settings.yaml" files that define settings for a given test case (i.e. translator feature flags), +// - a set of expected golden "_golden.yaml" files (in declarative config format) where each file represents an // expected output for a given translator configuration defined in "_settings.yaml". // -// The test case is executed by loading the in.yaml file into the store, then running the translator on the store, -// and finally comparing the output of the translator with the expected golden file. +// The test case is executed by loading the in.yaml file into the store, then running KongClient.Update method with +// the store injected. KongClient pushes configuration to a mock Admin API HTTP server. We fetch the last received +// configuration from the server and compare the output with the expected golden file. // // When adding a new test case, you can follow these steps: // 1. Add a new directory ./testdata/golden/ with the "in.yaml" that you want to test against. @@ -80,9 +83,9 @@ func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService // // If you introduce a change that may affect many test cases, and you're sure about it correctness, you can run the // update command as well to update all golden files at once. -func TestTranslator_GoldenTests(t *testing.T) { +func TestKongClient_GoldenTests(t *testing.T) { // First, let's prepare the test cases basing on the testdata/golden directory contents. - var testCases []translatorGoldenTestCase + var testCases []kongClientGoldenTestCase testCasesDirectories, err := os.ReadDir(goldenDir) require.NoError(t, err, "failed to iterate over files in testdata/golden") @@ -98,18 +101,18 @@ func TestTranslator_GoldenTests(t *testing.T) { // Then, let's iterate over all settings files in the directory and add a test case for each of them. // If there are no settings files, we'll add just a single test case with default settings. - for _, translatorSettings := range resolveSetsOfTranslatorSettingsForTestCaseDir(t, testCaseDirPath) { - testCases = append(testCases, translatorGoldenTestCase{ + for _, settings := range resolveSetsOfSettingsForTestCaseDir(t, testCaseDirPath) { + testCases = append(testCases, kongClientGoldenTestCase{ k8sConfigFile: filepath.Join(testCaseDirPath, inFileName), - goldenFile: filepath.Join(testCaseDirPath, fmt.Sprintf("%s%s", translatorSettings.name, goldenFileSuffix)), - featureFlags: translatorSettings.featureFlags, + goldenFile: filepath.Join(testCaseDirPath, fmt.Sprintf("%s%s", settings.name, goldenFileSuffix)), + featureFlags: settings.featureFlags, }) } } for _, tc := range testCases { t.Run(fmt.Sprintf("in=%s,out=%s", tc.k8sConfigFile, tc.goldenFile), func(t *testing.T) { - runTranslatorGoldenTest(t, tc) + runKongClientGoldenTest(t, tc) }) } } @@ -131,24 +134,26 @@ func pruneTestCaseDirectory(t *testing.T, path string) { } } -// resolveSetsOfTranslatorSettingsForTestCaseDir returns a slice of translatorSettings, each of which represents a combination of +// resolveSetsOfSettingsForTestCaseDir returns a slice of testCaseSettings, each of which represents a combination of // feature flags and Kong version. // The function iterates over a test case directory containing zero or more files named "_settings.yaml". -// If it doesn't find any settings files, it returns a single translatorSettings with default feature flags and Kong version. -func resolveSetsOfTranslatorSettingsForTestCaseDir(t *testing.T, path string) []translatorSettings { +// If it doesn't find any settings files, it returns a single testCaseSettings with default feature flags and Kong version. +func resolveSetsOfSettingsForTestCaseDir(t *testing.T, path string) []testCaseSettings { + t.Helper() + // Iterate over all files in the directory and look for settings files. files, err := os.ReadDir(path) require.NoErrorf(t, err, "failed to iterate over files in test case directory %s", path) - setsOfTranslatorSettings := []translatorSettings{ - // Always include a translatorSettings with default feature flags and Kong version. + setsOfTranslatorSettings := []testCaseSettings{ + // Always include a testCaseSettings with default feature flags and Kong version. { name: "default", featureFlags: defaultFeatureFlags(), }, } - // Iterate over all settings files and create a translatorSettings for each. + // Iterate over all settings files and create a testCaseSettings for each. for _, file := range files { require.False(t, file.IsDir(), "unexpected directory %s in test case directory %s", file.Name(), path) @@ -168,14 +173,14 @@ func resolveSetsOfTranslatorSettingsForTestCaseDir(t *testing.T, path string) [] return setsOfTranslatorSettings } -type translatorSettings struct { +type testCaseSettings struct { name string featureFlags translator.FeatureFlags } -// unmarshalSettingsFile unmarshals a settings file and returns a translatorSettings struct. +// unmarshalSettingsFile unmarshals a settings file and returns a testCaseSettings struct. // All feature flags and Kong version specified in the settings file will be used to override the defaults. -func unmarshalSettingsFile(t *testing.T, path string) translatorSettings { +func unmarshalSettingsFile(t *testing.T, path string) testCaseSettings { // It specifies only the json tags, because we're using "sigs.k8s.io/yaml" to unmarshal the file and that // package respects only json tags: "Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object". type settingsFile struct { @@ -203,22 +208,29 @@ func unmarshalSettingsFile(t *testing.T, path string) translatorSettings { field.SetBool(featureFlagValue) } - return translatorSettings{ + return testCaseSettings{ name: settingsName, featureFlags: featureFlags, } } -// translatorGoldenTestCase represents a single test case for the translator with an input file and an expected output golden -// file for a specific combination of feature flags and Kong version. -type translatorGoldenTestCase struct { +// kongClientGoldenTestCase represents a single test case for the KongClient with an input file and an expected output golden +// file for a specific combination of feature flags. +type kongClientGoldenTestCase struct { + // k8sConfigFile is the path to the input file with K8s objects to be loaded into the store. k8sConfigFile string - goldenFile string - featureFlags translator.FeatureFlags + // goldenFile is the path to the expected output golden file. + goldenFile string + // featureFlags is the set of Translator feature flags to use in the test case. + featureFlags translator.FeatureFlags } -func runTranslatorGoldenTest(t *testing.T, tc translatorGoldenTestCase) { - logger := zapr.NewLogger(zap.NewNop()) +// runKongClientGoldenTest runs a single golden test case for the KongClient. +func runKongClientGoldenTest(t *testing.T, tc kongClientGoldenTestCase) { + t.Helper() + + t.Logf("Running test case with input file %s and golden file %s", tc.k8sConfigFile, tc.goldenFile) + t.Logf("Feature flags: %+v", tc.featureFlags) // Load the K8s objects from the YAML file. objects := extractObjectsFromYAML(t, tc.k8sConfigFile) @@ -229,24 +241,60 @@ func runTranslatorGoldenTest(t *testing.T, tc translatorGoldenTestCase) { require.NoError(t, err, "failed creating cache stores") // Create the translator. + logger := zapr.NewLogger(zap.NewNop()) s := store.New(cacheStores, "kong", logger) p, err := translator.NewTranslator(logger, s, "", tc.featureFlags, fakeSchemaServiceProvier{}) require.NoError(t, err, "failed creating translator") - // MustBuild the Kong configuration. - result := p.BuildKongConfig() - targetConfig := deckgen.ToDeckContent(context.Background(), + // Start a mock Admin API server and create an Admin API client for inspecting the configuration. + t.Log("Starting mock Admin API server") + adminAPIHandler := mocks.NewAdminAPIHandler(t) + adminAPIServer := httptest.NewServer(adminAPIHandler) + defer adminAPIServer.Close() + + t.Log("Creating Admin API client") + adminAPIClient, err := adminapi.NewTestClient(adminAPIServer.URL) + require.NoError(t, err) + + // Create the KongClient using _mostly_ real dependencies' implementations (except for the clients provider + // as we want to avoid spinning up a real Kong Gateway to keep the tests fast). + t.Log("Building KongClient") + const timeout = time.Second + cfg := sendconfig.Config{ + InMemory: true, // We're running in DB-less mode only for now. In the future, we may want to test DB mode as well. + ExpressionRoutes: tc.featureFlags.ExpressionRoutes, + } + clientsProvider := &mockGatewayClientsProvider{ + gatewayClients: []*adminapi.Client{adminAPIClient}, + } + updateStrategyResolver := sendconfig.NewDefaultUpdateStrategyResolver(cfg, logger) + lastValidConfigFetcher := configfetcher.NewDefaultKongLastGoodConfigFetcher(tc.featureFlags.FillIDs, "default") + fallbackConfigGenerator := fallback.NewGenerator(fallback.NewDefaultCacheGraphProvider(), logger) + kongClient, err := NewKongClient( logger, - result.KongState, - deckgen.GenerateDeckContentParams{ - ExpressionRoutes: tc.featureFlags.ExpressionRoutes, - PluginSchemas: pluginsSchemaStoreStub{}, - }, + timeout, + diagnostics.ConfigDumpDiagnostic{}, + cfg, + mocks.NewEventRecorder(), + dpconf.DBModeOff, // Test will run in DB-less mode only for now. In the future, we may want to test DB mode as well. + clientsProvider, + updateStrategyResolver, + sendconfig.NewDefaultConfigurationChangeDetector(logger), + lastValidConfigFetcher, + p, + cacheStores, + fallbackConfigGenerator, ) + require.NoError(t, err) - // Marshal the result into YAML bytes for comparison. - resultB, err := yaml.Marshal(targetConfig) - require.NoError(t, err, "failed marshalling result") + t.Log("Triggering KongClient.Update") + ctx := context.Background() + err = kongClient.Update(ctx) + require.NoError(t, err, "failed updating Kong configuration") + + t.Log("Fetching the last received configuration from the Admin API") + resultB, err := adminAPIClient.AdminAPIClient().Config(ctx) + require.NoError(t, err) // If the update flag is set, update the golden file with the result... if *updateGolden { @@ -288,10 +336,10 @@ func extractObjectsFromYAML(t *testing.T, filePath string) [][]byte { }) } -// pluginsSchemaStoreStub is a stub implementation of the plugins.SchemaStore interface that returns an empty schema -// for all plugins. It's used to avoid hitting the Kong Admin API during tests. -type pluginsSchemaStoreStub struct{} +// fakeSchemaServiceProvier is a stub implementation of the SchemaServiceProvider interface that returns an +// UnavailableSchemaService. It's used to avoid hitting the Kong Admin API during tests. +type fakeSchemaServiceProvier struct{} -func (p pluginsSchemaStoreStub) Schema(context.Context, string) (map[string]interface{}, error) { - return map[string]interface{}{}, nil +func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService { + return translator.UnavailableSchemaService{} } diff --git a/internal/dataplane/translator/testdata/golden/consumer-group-example-ee/default_golden.yaml b/internal/dataplane/testdata/golden/consumer-group-example-ee/default_golden.yaml similarity index 71% rename from internal/dataplane/translator/testdata/golden/consumer-group-example-ee/default_golden.yaml rename to internal/dataplane/testdata/golden/consumer-group-example-ee/default_golden.yaml index eefc6c8017..3d18636028 100644 --- a/internal/dataplane/translator/testdata/golden/consumer-group-example-ee/default_golden.yaml +++ b/internal/dataplane/testdata/golden/consumer-group-example-ee/default_golden.yaml @@ -1,4 +1,11 @@ _format_version: "3.0" +consumer_group_consumers: +- consumer: consumer-2 + consumer_group: consumer-group-1 +- consumer: consumer-2 + consumer_group: consumer-group-2 +- consumer: consumer-1 + consumer_group: consumer-group-1 consumer_groups: - id: 876a1c78-f75a-570e-a918-26506344add0 name: consumer-group-2 @@ -15,19 +22,14 @@ consumer_groups: - k8s-group:configuration.konghq.com - k8s-version:v1beta1 consumers: -- groups: - - name: consumer-group-1 - - name: consumer-group-2 - id: 71168e5f-1d0b-5465-8bb9-a3b032fbc4c4 +- id: 71168e5f-1d0b-5465-8bb9-a3b032fbc4c4 tags: - k8s-name:consumer-2 - k8s-kind:KongConsumer - k8s-group:configuration.konghq.com - k8s-version:v1 username: consumer-2 -- groups: - - name: consumer-group-1 - id: e23d9ef8-1cc4-5b55-abd1-db89aa90346c +- id: e23d9ef8-1cc4-5b55-abd1-db89aa90346c tags: - k8s-name:consumer-1 - k8s-kind:KongConsumer diff --git a/internal/dataplane/translator/testdata/golden/consumer-group-example-ee/in.yaml b/internal/dataplane/testdata/golden/consumer-group-example-ee/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/consumer-group-example-ee/in.yaml rename to internal/dataplane/testdata/golden/consumer-group-example-ee/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/grpcroute-example/default_golden.yaml b/internal/dataplane/testdata/golden/grpcroute-example/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/grpcroute-example/default_golden.yaml rename to internal/dataplane/testdata/golden/grpcroute-example/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/grpcroute-example/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/grpcroute-example/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/grpcroute-example/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/grpcroute-example/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/grpcroute-example/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/grpcroute-example/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/grpcroute-example/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/grpcroute-example/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/grpcroute-example/in.yaml b/internal/dataplane/testdata/golden/grpcroute-example/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/grpcroute-example/in.yaml rename to internal/dataplane/testdata/golden/grpcroute-example/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/host-header-annotation-httproute/default_golden.yaml b/internal/dataplane/testdata/golden/host-header-annotation-httproute/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/host-header-annotation-httproute/default_golden.yaml rename to internal/dataplane/testdata/golden/host-header-annotation-httproute/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/host-header-annotation-httproute/in.yaml b/internal/dataplane/testdata/golden/host-header-annotation-httproute/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/host-header-annotation-httproute/in.yaml rename to internal/dataplane/testdata/golden/host-header-annotation-httproute/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-example/default_golden.yaml b/internal/dataplane/testdata/golden/httproute-example/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-example/default_golden.yaml rename to internal/dataplane/testdata/golden/httproute-example/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-example/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/httproute-example/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-example/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/httproute-example/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-example/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/httproute-example/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-example/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/httproute-example/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-example/in.yaml b/internal/dataplane/testdata/golden/httproute-example/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-example/in.yaml rename to internal/dataplane/testdata/golden/httproute-example/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/default_golden.yaml b/internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/default_golden.yaml rename to internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/in.yaml b/internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/httproute-url-rewrite-path-prefix/in.yaml rename to internal/dataplane/testdata/golden/httproute-url-rewrite-path-prefix/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-empty-path/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-empty-path/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-empty-path/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-empty-path/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-empty-path/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-empty-path/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-empty-path/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-empty-path/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-empty-path/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ingress-class-annotation/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-ingress-class-annotation/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ingress-class-annotation/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ingress-class-annotation/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ingress-class-annotation/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-ingress-class-annotation/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ingress-class-annotation/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ingress-class-annotation/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-multiple-ports-for-one-service/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-multiple-ports-for-one-service/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-ports-defined-by-name/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-ports-defined-by-name/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefix-exact-rule/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefix-exact-rule/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-regex-prefixed-path/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-regex-prefixed-path/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-rule-with-tls/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-rule-with-tls/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-single-service-in-multiple-ingresses/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-acme-like-path/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-acme-like-path/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/default_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-default-backend/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/default_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-default-backend/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-default-backend/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/in.yaml b/internal/dataplane/testdata/golden/ingress-v1-with-default-backend/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/ingress-v1-with-default-backend/in.yaml rename to internal/dataplane/testdata/golden/ingress-v1-with-default-backend/in.yaml diff --git a/internal/dataplane/testdata/golden/kong-service-facade/default_golden.yaml b/internal/dataplane/testdata/golden/kong-service-facade/default_golden.yaml new file mode 100644 index 0000000000..017467254d --- /dev/null +++ b/internal/dataplane/testdata/golden/kong-service-facade/default_golden.yaml @@ -0,0 +1,3 @@ +_format_version: "3.0" +upstreams: +- name: kong diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/expression-routes-on_golden.yaml b/internal/dataplane/testdata/golden/kong-service-facade/expression-routes-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kong-service-facade/expression-routes-on_golden.yaml rename to internal/dataplane/testdata/golden/kong-service-facade/expression-routes-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/expression-routes-on_settings.yaml b/internal/dataplane/testdata/golden/kong-service-facade/expression-routes-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kong-service-facade/expression-routes-on_settings.yaml rename to internal/dataplane/testdata/golden/kong-service-facade/expression-routes-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/feature-flag-on_golden.yaml b/internal/dataplane/testdata/golden/kong-service-facade/feature-flag-on_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kong-service-facade/feature-flag-on_golden.yaml rename to internal/dataplane/testdata/golden/kong-service-facade/feature-flag-on_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/feature-flag-on_settings.yaml b/internal/dataplane/testdata/golden/kong-service-facade/feature-flag-on_settings.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kong-service-facade/feature-flag-on_settings.yaml rename to internal/dataplane/testdata/golden/kong-service-facade/feature-flag-on_settings.yaml diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/in.yaml b/internal/dataplane/testdata/golden/kong-service-facade/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kong-service-facade/in.yaml rename to internal/dataplane/testdata/golden/kong-service-facade/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongplugin-instance-name/default_golden.yaml b/internal/dataplane/testdata/golden/kongplugin-instance-name/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongplugin-instance-name/default_golden.yaml rename to internal/dataplane/testdata/golden/kongplugin-instance-name/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongplugin-instance-name/in.yaml b/internal/dataplane/testdata/golden/kongplugin-instance-name/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongplugin-instance-name/in.yaml rename to internal/dataplane/testdata/golden/kongplugin-instance-name/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongupstreampolicy-httproute/default_golden.yaml b/internal/dataplane/testdata/golden/kongupstreampolicy-httproute/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongupstreampolicy-httproute/default_golden.yaml rename to internal/dataplane/testdata/golden/kongupstreampolicy-httproute/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongupstreampolicy-httproute/in.yaml b/internal/dataplane/testdata/golden/kongupstreampolicy-httproute/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongupstreampolicy-httproute/in.yaml rename to internal/dataplane/testdata/golden/kongupstreampolicy-httproute/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongupstreampolicy-ingress/default_golden.yaml b/internal/dataplane/testdata/golden/kongupstreampolicy-ingress/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongupstreampolicy-ingress/default_golden.yaml rename to internal/dataplane/testdata/golden/kongupstreampolicy-ingress/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/kongupstreampolicy-ingress/in.yaml b/internal/dataplane/testdata/golden/kongupstreampolicy-ingress/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/kongupstreampolicy-ingress/in.yaml rename to internal/dataplane/testdata/golden/kongupstreampolicy-ingress/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/same-name-services-single-backend/default_golden.yaml b/internal/dataplane/testdata/golden/same-name-services-single-backend/default_golden.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/same-name-services-single-backend/default_golden.yaml rename to internal/dataplane/testdata/golden/same-name-services-single-backend/default_golden.yaml diff --git a/internal/dataplane/translator/testdata/golden/same-name-services-single-backend/in.yaml b/internal/dataplane/testdata/golden/same-name-services-single-backend/in.yaml similarity index 100% rename from internal/dataplane/translator/testdata/golden/same-name-services-single-backend/in.yaml rename to internal/dataplane/testdata/golden/same-name-services-single-backend/in.yaml diff --git a/internal/dataplane/translator/testdata/golden/kong-service-facade/default_golden.yaml b/internal/dataplane/translator/testdata/golden/kong-service-facade/default_golden.yaml deleted file mode 100644 index 60dfed1f9a..0000000000 --- a/internal/dataplane/translator/testdata/golden/kong-service-facade/default_golden.yaml +++ /dev/null @@ -1 +0,0 @@ -_format_version: "3.0" diff --git a/test/kongintegration/translator_golden_tests_outputs_test.go b/test/kongintegration/kong_client_golden_tests_outputs_test.go similarity index 71% rename from test/kongintegration/translator_golden_tests_outputs_test.go rename to test/kongintegration/kong_client_golden_tests_outputs_test.go index 27331724d6..9c777db235 100644 --- a/test/kongintegration/translator_golden_tests_outputs_test.go +++ b/test/kongintegration/kong_client_golden_tests_outputs_test.go @@ -1,7 +1,9 @@ package kongintegration import ( + "bytes" "context" + "encoding/json" "os" "path/filepath" "strings" @@ -12,7 +14,9 @@ import ( "github.com/go-logr/logr" "github.com/kong/go-database-reconciler/pkg/dump" "github.com/kong/go-database-reconciler/pkg/file" + "github.com/kong/go-kong/kong" "github.com/samber/lo" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" @@ -28,8 +32,8 @@ const ( tick = 100 * time.Millisecond ) -// TestGoldenTestsOutputs ensures that the translators' golden tests outputs are accepted by Kong. -func TestTranslatorsGoldenTestsOutputs(t *testing.T) { +// TestKongClientGoldenTestsOutputs ensures that the KongClient's golden tests outputs are accepted by Kong. +func TestKongClientGoldenTestsOutputs(t *testing.T) { t.Parallel() ctx := context.Background() @@ -56,15 +60,9 @@ func TestTranslatorsGoldenTestsOutputs(t *testing.T) { kongClient, err := adminapi.NewKongAPIClient(kongC.AdminURL(ctx, t), helpers.DefaultHTTPClient()) require.NoError(t, err) - sut := sendconfig.NewUpdateStrategyInMemory( - kongClient, - sendconfig.DefaultContentToDBLessConfigConverter{}, - logr.Discard(), - ) - for _, goldenTestOutputPath := range expressionRoutesOutputsPaths { t.Run(goldenTestOutputPath, func(t *testing.T) { - ensureGoldenTestOutputIsAccepted(ctx, t, goldenTestOutputPath, sut) + ensureGoldenTestOutputIsAccepted(ctx, t, goldenTestOutputPath, kongClient) }) } }) @@ -76,23 +74,17 @@ func TestTranslatorsGoldenTestsOutputs(t *testing.T) { kongClient, err := adminapi.NewKongAPIClient(kongC.AdminURL(ctx, t), helpers.DefaultHTTPClient()) require.NoError(t, err) - sut := sendconfig.NewUpdateStrategyInMemory( - kongClient, - sendconfig.DefaultContentToDBLessConfigConverter{}, - logr.Discard(), - ) - for _, goldenTestOutputPath := range defaultOutputsPaths { t.Run(goldenTestOutputPath, func(t *testing.T) { - ensureGoldenTestOutputIsAccepted(ctx, t, goldenTestOutputPath, sut) + ensureGoldenTestOutputIsAccepted(ctx, t, goldenTestOutputPath, kongClient) }) } }) } -// TestGoldenTestsOutputs ensures that the translators' golden tests outputs are accepted by Konnect Control Plane +// TestKongClientGoldenTestsOutputs ensures that the KongClient's golden tests outputs are accepted by Konnect Control Plane // Admin API. -func TestTranslatorsGoldenTestsOutputs_Konnect(t *testing.T) { +func TestKongClientGoldenTestsOutputs_Konnect(t *testing.T) { konnect.SkipIfMissingRequiredKonnectEnvVariables(t) t.Parallel() @@ -108,35 +100,40 @@ func TestTranslatorsGoldenTestsOutputs_Konnect(t *testing.T) { for _, goldenTestOutputPath := range allGoldenTestsOutputsPaths(t) { t.Run(goldenTestOutputPath, func(t *testing.T) { - ensureGoldenTestOutputIsAccepted(ctx, t, goldenTestOutputPath, updateStrategy) + goldenTestOutput, err := os.ReadFile(goldenTestOutputPath) + require.NoError(t, err) + + content := &file.Content{} + err = yaml.Unmarshal(goldenTestOutput, content) + require.NoError(t, err) + + require.EventuallyWithT(t, func(t *assert.CollectT) { + err := updateStrategy.Update(ctx, sendconfig.ContentWithHash{Content: content}) + assert.NoError(t, err) + }, timeout, tick) }) } } -func ensureGoldenTestOutputIsAccepted( - ctx context.Context, - t *testing.T, - goldenTestOutputPath string, - sut sendconfig.UpdateStrategy, -) { +func ensureGoldenTestOutputIsAccepted(ctx context.Context, t *testing.T, goldenTestOutputPath string, kongClient *kong.Client) { goldenTestOutput, err := os.ReadFile(goldenTestOutputPath) require.NoError(t, err) - content := &file.Content{} - err = yaml.Unmarshal(goldenTestOutput, content) + cfg := map[string]any{} + err = yaml.Unmarshal(goldenTestOutput, &cfg) require.NoError(t, err) - require.Eventually(t, func() bool { - if err := sut.Update(ctx, sendconfig.ContentWithHash{Content: content}); err != nil { - t.Logf("error: %v", err) - return false - } - return true + cfgAsJSON, err := json.Marshal(cfg) + require.NoError(t, err) + + require.EventuallyWithT(t, func(t *assert.CollectT) { + resp, err := kongClient.ReloadDeclarativeRawConfig(ctx, bytes.NewReader(cfgAsJSON), true, true) + assert.NoErrorf(t, err, "failed to reload declarative config, resp: %s", string(resp)) }, timeout, tick) } func allGoldenTestsOutputsPaths(t *testing.T) []string { - const goldenTestsOutputsGlob = "../../internal/dataplane/translator/testdata/golden/*/*_golden.yaml" + const goldenTestsOutputsGlob = "../../internal/dataplane/testdata/golden/*/*_golden.yaml" goldenTestsOutputsPaths, err := filepath.Glob(goldenTestsOutputsGlob) require.NoError(t, err) require.NotEmpty(t, goldenTestsOutputsPaths, "no golden tests outputs found") diff --git a/test/mocks/admin_api_handler.go b/test/mocks/admin_api_handler.go index 9ba3affd8e..c04c57abc5 100644 --- a/test/mocks/admin_api_handler.go +++ b/test/mocks/admin_api_handler.go @@ -1,12 +1,16 @@ package mocks import ( + "encoding/json" "fmt" "io" "net/http" "sync/atomic" "testing" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" + "github.com/kong/kubernetes-ingress-controller/v3/internal/versions" ) @@ -184,7 +188,7 @@ func NewAdminAPIHandler(t *testing.T, opts ...AdminAPIHandlerOpt) *AdminAPIHandl switch r.Method { case http.MethodGet: if h.config != nil { - _, _ = w.Write(h.config) + _, _ = w.Write(convertReceivedJSONConfigIntoGetConfigResponse(t, h.config)) } else { _, _ = w.Write([]byte(fmt.Sprintf(`{"version": "%s"}`, h.version))) } @@ -440,3 +444,23 @@ const defaultDBLessStatusResponseWithoutConfigurationHash = `{ "connections_active": 3 } }` + +// convertReceivedJSONConfigIntoGetConfigResponse converts the received JSON config into a response for a `GET /config` request. +// That's the way Kong Gateway behaves and to satisfy kong.Client expectations we need to do the same in the mock server. +func convertReceivedJSONConfigIntoGetConfigResponse(t *testing.T, jsonConfig []byte) []byte { + t.Helper() + + cfg := map[string]any{} + err := json.Unmarshal(jsonConfig, &cfg) + require.NoError(t, err, "failed unmarshalling result") + resultB, err := yaml.Marshal(cfg) + require.NoError(t, err, "failed marshalling result") + body := struct { + Config string `json:"config"` + }{ + Config: string(resultB), + } + respB, err := json.Marshal(body) + require.NoError(t, err) + return respB +} From 77af8dc7adb4cb37ce5763d48e4cc81618b5df46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 6 Jun 2024 22:46:27 +0200 Subject: [PATCH 45/48] chore(tests): add fallback config golden tests for Ingress and Service (#6157) --- internal/dataplane/kong_client_golden_test.go | 49 ++++++++++++-- .../default_golden.yaml | 48 ++++++++++++++ .../golden/fallback-config-ingress/in.yaml | 56 ++++++++++++++++ .../default_golden.yaml | 48 ++++++++++++++ .../golden/fallback-config-service/in.yaml | 64 +++++++++++++++++++ 5 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 internal/dataplane/testdata/golden/fallback-config-ingress/default_golden.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-ingress/in.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-service/default_golden.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-service/in.yaml diff --git a/internal/dataplane/kong_client_golden_test.go b/internal/dataplane/kong_client_golden_test.go index 877a3acaaa..3fc2393750 100644 --- a/internal/dataplane/kong_client_golden_test.go +++ b/internal/dataplane/kong_client_golden_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" "k8s.io/kubectl/pkg/cmd/util" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi" @@ -83,6 +84,10 @@ const ( // // If you introduce a change that may affect many test cases, and you're sure about it correctness, you can run the // update command as well to update all golden files at once. +// +// If you want to make the mocked Admin API server return errors for specific objects, you can add an annotation +// "test.konghq.com/broken: true" to the object in the in.yaml file. If there's at least one object with this annotation, +// the test will expect an error from the KongClient.Update method and will turn the FallbackConfiguration feature on. func TestKongClient_GoldenTests(t *testing.T) { // First, let's prepare the test cases basing on the testdata/golden directory contents. var testCases []kongClientGoldenTestCase @@ -240,6 +245,16 @@ func runKongClientGoldenTest(t *testing.T, tc kongClientGoldenTestCase) { cacheStores, err := store.NewCacheStoresFromObjYAML(objects...) require.NoError(t, err, "failed creating cache stores") + var objectsToBeConsideredBroken []fallback.ObjectHash + for _, s := range cacheStores.ListAllStores() { + for _, o := range s.List() { + o := o.(client.Object) + if o.GetAnnotations()["test.konghq.com/broken"] == "true" { + objectsToBeConsideredBroken = append(objectsToBeConsideredBroken, fallback.GetObjectHash(o)) + } + } + } + // Create the translator. logger := zapr.NewLogger(zap.NewNop()) s := store.New(cacheStores, "kong", logger) @@ -248,7 +263,16 @@ func runKongClientGoldenTest(t *testing.T, tc kongClientGoldenTestCase) { // Start a mock Admin API server and create an Admin API client for inspecting the configuration. t.Log("Starting mock Admin API server") - adminAPIHandler := mocks.NewAdminAPIHandler(t) + var adminAPIOpts []mocks.AdminAPIHandlerOpt + if len(objectsToBeConsideredBroken) > 0 { + t.Logf("Configuring the mock Admin API server to return errors for broken objects: %v", objectsToBeConsideredBroken) + adminAPIOpts = append(adminAPIOpts, + mocks.WithConfigPostError(buildPostConfigErrorResponseWithBrokenObjects(objectsToBeConsideredBroken)), + mocks.WithConfigPostErrorOnlyOnFirstRequest(), + ) + } + + adminAPIHandler := mocks.NewAdminAPIHandler(t, adminAPIOpts...) adminAPIServer := httptest.NewServer(adminAPIHandler) defer adminAPIServer.Close() @@ -261,8 +285,9 @@ func runKongClientGoldenTest(t *testing.T, tc kongClientGoldenTestCase) { t.Log("Building KongClient") const timeout = time.Second cfg := sendconfig.Config{ - InMemory: true, // We're running in DB-less mode only for now. In the future, we may want to test DB mode as well. - ExpressionRoutes: tc.featureFlags.ExpressionRoutes, + InMemory: true, // We're running in DB-less mode only for now. In the future, we may want to test DB mode as well. + ExpressionRoutes: tc.featureFlags.ExpressionRoutes, + FallbackConfiguration: len(objectsToBeConsideredBroken) > 0, } clientsProvider := &mockGatewayClientsProvider{ gatewayClients: []*adminapi.Client{adminAPIClient}, @@ -290,7 +315,11 @@ func runKongClientGoldenTest(t *testing.T, tc kongClientGoldenTestCase) { t.Log("Triggering KongClient.Update") ctx := context.Background() err = kongClient.Update(ctx) - require.NoError(t, err, "failed updating Kong configuration") + if len(objectsToBeConsideredBroken) > 0 { + require.Error(t, err, "expected an error when fallback configuration is enabled") + } else { + require.NoError(t, err, "failed updating Kong configuration") + } t.Log("Fetching the last received configuration from the Admin API") resultB, err := adminAPIClient.AdminAPIClient().Config(ctx) @@ -343,3 +372,15 @@ type fakeSchemaServiceProvier struct{} func (p fakeSchemaServiceProvier) GetSchemaService() kong.AbstractSchemaService { return translator.UnavailableSchemaService{} } + +func buildPostConfigErrorResponseWithBrokenObjects(brokenObjects []fallback.ObjectHash) []byte { + var flattenedErrors []string + for _, o := range brokenObjects { + flattenedError := fmt.Sprintf(`{"errors": [{"messages": ["broken object"]}], "entity_tags": ["k8s-name:%s","k8s-namespace:%s","k8s-kind:%s","k8s-group:%s", "k8s-uid:%s"]}`, + o.Name, o.Namespace, o.Kind, o.Group, o.UID, + ) + flattenedErrors = append(flattenedErrors, flattenedError) + } + + return []byte(fmt.Sprintf(`{"flattened_errors": [%s]}`, strings.Join(flattenedErrors, ","))) +} diff --git a/internal/dataplane/testdata/golden/fallback-config-ingress/default_golden.yaml b/internal/dataplane/testdata/golden/fallback-config-ingress/default_golden.yaml new file mode 100644 index 0000000000..ad718196fb --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-ingress/default_golden.yaml @@ -0,0 +1,48 @@ +_format_version: "3.0" +services: +- connect_timeout: 60000 + host: svc.foo-namespace.80.svc + id: b39d28b5-b340-5fdf-951c-7533171d95bb + name: foo-namespace.svc.80 + path: / + port: 80 + protocol: http + read_timeout: 60000 + retries: 5 + routes: + - hosts: + - example.com + https_redirect_status_code: 426 + id: 6ac99766-f0f3-51ec-ae0f-df376a67738e + name: foo-namespace.valid-ingress.svc.example.com.80 + path_handling: v0 + paths: + - ~/valid$ + preserve_host: true + protocols: + - http + - https + regex_priority: 0 + request_buffering: true + response_buffering: true + strip_path: false + tags: + - k8s-name:valid-ingress + - k8s-namespace:foo-namespace + - k8s-kind:Ingress + - k8s-group:networking.k8s.io + - k8s-version:v1 + tags: + - k8s-name:svc + - k8s-namespace:foo-namespace + - k8s-kind:Service + - k8s-version:v1 + write_timeout: 60000 +upstreams: +- algorithm: round-robin + name: svc.foo-namespace.80.svc + tags: + - k8s-name:svc + - k8s-namespace:foo-namespace + - k8s-kind:Service + - k8s-version:v1 diff --git a/internal/dataplane/testdata/golden/fallback-config-ingress/in.yaml b/internal/dataplane/testdata/golden/fallback-config-ingress/in.yaml new file mode 100644 index 0000000000..4b3ab9b4ba --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-ingress/in.yaml @@ -0,0 +1,56 @@ +# In this test case we have two Ingresses and one Service. One of the Ingresses is broken and the other is valid. +# We expect the broken Ingress to be excluded. +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: broken-ingress + namespace: foo-namespace + uid: "6faea5d6-ce95-439e-b223-421a0a142e3f" + annotations: + test.konghq.com/broken: "true" +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: svc + port: + number: 80 + path: /broken + pathType: Exact +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: kong + name: valid-ingress + namespace: foo-namespace +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: svc + port: + number: 80 + path: /valid + pathType: Exact +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + kubernetes.io/ingress.class: kong + name: svc + namespace: foo-namespace +spec: + ports: + - port: 80 diff --git a/internal/dataplane/testdata/golden/fallback-config-service/default_golden.yaml b/internal/dataplane/testdata/golden/fallback-config-service/default_golden.yaml new file mode 100644 index 0000000000..1910735a35 --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-service/default_golden.yaml @@ -0,0 +1,48 @@ +_format_version: "3.0" +services: +- connect_timeout: 60000 + host: valid-svc.foo-namespace.80.svc + id: 7ef49b77-070d-522e-bb48-b13885406ee7 + name: foo-namespace.valid-svc.80 + path: / + port: 80 + protocol: http + read_timeout: 60000 + retries: 5 + routes: + - hosts: + - example.com + https_redirect_status_code: 426 + id: 5e522fc0-aa10-5a7c-a220-1521a77295cd + name: foo-namespace.valid-ingress.valid-svc.example.com.80 + path_handling: v0 + paths: + - ~/valid$ + preserve_host: true + protocols: + - http + - https + regex_priority: 0 + request_buffering: true + response_buffering: true + strip_path: false + tags: + - k8s-name:valid-ingress + - k8s-namespace:foo-namespace + - k8s-kind:Ingress + - k8s-group:networking.k8s.io + - k8s-version:v1 + tags: + - k8s-name:valid-svc + - k8s-namespace:foo-namespace + - k8s-kind:Service + - k8s-version:v1 + write_timeout: 60000 +upstreams: +- algorithm: round-robin + name: valid-svc.foo-namespace.80.svc + tags: + - k8s-name:valid-svc + - k8s-namespace:foo-namespace + - k8s-kind:Service + - k8s-version:v1 diff --git a/internal/dataplane/testdata/golden/fallback-config-service/in.yaml b/internal/dataplane/testdata/golden/fallback-config-service/in.yaml new file mode 100644 index 0000000000..a7907ca809 --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-service/in.yaml @@ -0,0 +1,64 @@ +# In this test case, we have two Services and two Ingresses. One of the Services is broken and the other is valid. +# Because the broken Service is referenced in an Ingress, we expect the Ingress to be excluded. +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-with-broken-service + namespace: foo-namespace + uid: "6faea5d6-ce95-439e-b223-421a0a142e3f" +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: broken-svc + port: + number: 80 + path: /broken + pathType: Exact +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: kong + name: valid-ingress + namespace: foo-namespace +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: valid-svc + port: + number: 80 + path: /valid + pathType: Exact +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + test.konghq.com/broken: "true" + name: broken-svc + namespace: foo-namespace + uid: "6faea5d6-ce95-439e-b223-421a0a142e3f" +spec: + ports: + - port: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: valid-svc + namespace: foo-namespace +spec: + ports: + - port: 80 From 28567d76cc104e582225c3409fec08c0d0a3e2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Fri, 7 Jun 2024 09:15:55 +0200 Subject: [PATCH 46/48] chore: fix renovate config (#6154) --- config/image/enterprise/kustomization.yaml | 4 ++-- config/image/oss/kustomization.yaml | 4 ++-- renovate.json | 22 ++++++++++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/config/image/enterprise/kustomization.yaml b/config/image/enterprise/kustomization.yaml index 0db707f835..25de87b78a 100644 --- a/config/image/enterprise/kustomization.yaml +++ b/config/image/enterprise/kustomization.yaml @@ -6,7 +6,7 @@ kind: Component images: - name: kong newName: kong/kong-gateway - newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway@regenerate - name: kong-placeholder newName: kong/kong-gateway - newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong/kong-gateway@regenerate diff --git a/config/image/oss/kustomization.yaml b/config/image/oss/kustomization.yaml index e2a0681e08..190c5d8333 100644 --- a/config/image/oss/kustomization.yaml +++ b/config/image/oss/kustomization.yaml @@ -4,7 +4,7 @@ kind: Component images: - name: kong-placeholder newName: kong - newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong + newTag: '3.7' # renovate: datasource=docker versioning=docker depName=kong@regenerate - name: kic-placeholder newName: kong/kubernetes-ingress-controller - newTag: '3.1' # renovate: datasource=docker versioning=docker depName=kong/kubernetes-ingress-controller + newTag: '3.1' # renovate: datasource=docker versioning=docker depName=kong/kubernetes-ingress-controller@regenerate diff --git a/renovate.json b/renovate.json index 539aa24f0d..7a1bdcd0b7 100644 --- a/renovate.json +++ b/renovate.json @@ -34,6 +34,17 @@ "matchStrings": [ "# renovate: datasource=(?.*?) depName=(?.*?)\\n.+\"(?.*?)\"" ] + }, + { + "description": "Match versions in config/image/oss and config/image/enterprise kustomize files that are properly annotated with `# renovate: datasource={} versioning={} depName={}`.", + "customType": "regex", + "fileMatch": [ + "^config/image/enterprise/.*\\.yaml$", + "^config/image/oss/.*\\.yaml$" + ], + "matchStrings": [ + "'(?.+)' # renovate: datasource=(?.*) versioning=(?.*) depName=(?.+)" + ] } ], "customDatasources": { @@ -63,14 +74,9 @@ ] }, { - "description": "Match versions in config/image/oss and config/image/enterprise kustomize files that are properly annotated with `# renovate: datasource={} versioning={} depName={}`.", - "customType": "regex", - "fileMatch": [ - "^config/image/enterprise/.*\\.yaml$", - "^config/image/oss/.*\\.yaml$" - ], - "matchStrings": [ - "'(?.+)' # renovate: datasource=(?.*) versioning=(?.*) depName=(?.+)" + "description": "Add 'renovate/auto-regenerate' label to a PR if it changes kustomize files containing images to trigger regenerate_on_deps_bump.yaml workflow.", + "matchDepPatterns": [ + ".*@regenerate" ], "addLabels": [ "renovate/auto-regenerate" From ee797b4e84bd176526af32ab6db54f16ee9c245b Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 7 Jun 2024 01:15:48 -0700 Subject: [PATCH 47/48] fix(plugins) handle CG plugins like consumers (#6132) Use logic similar to consumers for consumer groups. This fixes an issue where plugins assigned to a consumer group and a route (or service) would not actually be associated with the consumer group, only the route (or service). --- CHANGELOG.md | 5 + internal/util/relations.go | 53 +++++++-- internal/util/relations_test.go | 112 +++++++++++++++---- test/conformance/gateway_conformance_test.go | 2 +- test/integration/consumer_group_test.go | 98 ++++++++++++++-- 5 files changed, 229 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01a87bef6..1e851977c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -225,6 +225,11 @@ Adding a new version? You'll need three changes: - Fixed KIC non leaders correctly getting up to date Admin API addresses by not requiring leader election for the related controller. [#6126](https://github.com/Kong/kubernetes-ingress-controller/pull/6126) +- Plugins attached to both a KongConsumerGroup and a route-like resource or + Service now properly generate a plugin attached to both a Kong consumer group + and route or service. Previously, these incorrectly generated plugins + attached to the route or service only. + [#6132](https://github.com/Kong/kubernetes-ingress-controller/pull/6132) - KongPlugin's `config` field is no longer incorrectly sanitized. [#6138](https://github.com/Kong/kubernetes-ingress-controller/pull/6138) diff --git a/internal/util/relations.go b/internal/util/relations.go index a0f9fc0cd5..bc923d8c10 100644 --- a/internal/util/relations.go +++ b/internal/util/relations.go @@ -9,36 +9,67 @@ type Rel struct { } func (relations *ForeignRelations) GetCombinations() []Rel { + var ( + lConsumer = len(relations.Consumer) + lConsumerGroup = len(relations.ConsumerGroup) + lRoutes = len(relations.Route) + lServices = len(relations.Service) + l = lRoutes + lServices + ) + var cartesianProduct []Rel - if len(relations.Consumer) > 0 { - consumers := relations.Consumer - if len(relations.Route)+len(relations.Service) > 0 { - for _, service := range relations.Service { - for _, consumer := range consumers { + // gocritic I don't care that you think switch statements are the one true god of readability, the language offers + // multiple options for a reason. go away, gocritic. + if lConsumer > 0 { //nolint:gocritic + if l > 0 { + cartesianProduct = make([]Rel, 0, l*lConsumer) + for _, consumer := range relations.Consumer { + for _, service := range relations.Service { cartesianProduct = append(cartesianProduct, Rel{ Service: service, Consumer: consumer, }) } - } - for _, route := range relations.Route { - for _, consumer := range consumers { + for _, route := range relations.Route { cartesianProduct = append(cartesianProduct, Rel{ Route: route, Consumer: consumer, }) } } + } else { + cartesianProduct = make([]Rel, 0, len(relations.Consumer)) for _, consumer := range relations.Consumer { cartesianProduct = append(cartesianProduct, Rel{Consumer: consumer}) } } - } else { - for _, consumerGroup := range relations.ConsumerGroup { - cartesianProduct = append(cartesianProduct, Rel{ConsumerGroup: consumerGroup}) + } else if lConsumerGroup > 0 { + if l > 0 { + cartesianProduct = make([]Rel, 0, l*lConsumerGroup) + for _, group := range relations.ConsumerGroup { + for _, service := range relations.Service { + cartesianProduct = append(cartesianProduct, Rel{ + Service: service, + ConsumerGroup: group, + }) + } + for _, route := range relations.Route { + cartesianProduct = append(cartesianProduct, Rel{ + Route: route, + ConsumerGroup: group, + }) + } + } + } else { + cartesianProduct = make([]Rel, 0, lConsumerGroup) + for _, group := range relations.ConsumerGroup { + cartesianProduct = append(cartesianProduct, Rel{ConsumerGroup: group}) + } } + } else if l > 0 { + cartesianProduct = make([]Rel, 0, l) for _, service := range relations.Service { cartesianProduct = append(cartesianProduct, Rel{Service: service}) } diff --git a/internal/util/relations_test.go b/internal/util/relations_test.go index 1f9dbff416..1e76640d19 100644 --- a/internal/util/relations_test.go +++ b/internal/util/relations_test.go @@ -1,8 +1,9 @@ package util import ( - "reflect" "testing" + + "github.com/stretchr/testify/require" ) func TestGetCombinations(t *testing.T) { @@ -121,14 +122,14 @@ func TestGetCombinations(t *testing.T) { Consumer: "foo", Route: "foo", }, - { - Consumer: "bar", - Route: "foo", - }, { Consumer: "foo", Route: "bar", }, + { + Consumer: "bar", + Route: "foo", + }, { Consumer: "bar", Route: "bar", @@ -148,14 +149,14 @@ func TestGetCombinations(t *testing.T) { Consumer: "foo", Service: "foo", }, - { - Consumer: "bar", - Service: "foo", - }, { Consumer: "foo", Service: "bar", }, + { + Consumer: "bar", + Service: "foo", + }, { Consumer: "bar", Service: "bar", @@ -177,41 +178,110 @@ func TestGetCombinations(t *testing.T) { Service: "s1", }, { - Consumer: "c2", - Service: "s1", + Consumer: "c1", + Service: "s2", }, { Consumer: "c1", - Service: "s2", + Route: "r1", + }, + { + Consumer: "c1", + Route: "r2", }, { Consumer: "c2", - Service: "s2", + Service: "s1", }, { - Consumer: "c1", - Route: "r1", + Consumer: "c2", + Service: "s2", }, { Consumer: "c2", Route: "r1", }, { - Consumer: "c1", + Consumer: "c2", Route: "r2", }, + }, + }, + { + name: "plugins on combination of service,route and consumer group", + args: args{ + relations: ForeignRelations{ + Route: []string{"r1", "r2"}, + Service: []string{"s1", "s2"}, + ConsumerGroup: []string{"cg1", "cg2"}, + }, + }, + want: []Rel{ { - Consumer: "c2", - Route: "r2", + ConsumerGroup: "cg1", + Service: "s1", + }, + { + ConsumerGroup: "cg1", + Service: "s2", + }, + { + ConsumerGroup: "cg1", + Route: "r1", + }, + { + ConsumerGroup: "cg1", + Route: "r2", + }, + { + ConsumerGroup: "cg2", + Service: "s1", + }, + { + ConsumerGroup: "cg2", + Service: "s2", + }, + { + ConsumerGroup: "cg2", + Route: "r1", + }, + { + ConsumerGroup: "cg2", + Route: "r2", }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := tt.args.relations.GetCombinations(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetCombinations() = %v, want %v", got, tt.want) - } + require.Equal(t, tt.want, tt.args.relations.GetCombinations()) }) } } + +func BenchmarkGetCombinations(b *testing.B) { + b.Run("consumer groups", func(b *testing.B) { + for i := 0; i < b.N; i++ { + relations := ForeignRelations{ + Route: []string{"r1", "r2"}, + Service: []string{"s1", "s2"}, + ConsumerGroup: []string{"cg1", "cg2"}, + } + + rels := relations.GetCombinations() + _ = rels + } + }) + b.Run("consumers", func(b *testing.B) { + for i := 0; i < b.N; i++ { + relations := ForeignRelations{ + Route: []string{"r1", "r2"}, + Service: []string{"s1", "s2"}, + Consumer: []string{"c1", "c2", "c3"}, + } + + rels := relations.GetCombinations() + _ = rels + } + }) +} diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index c7f89fc338..6e75d37b43 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -30,7 +30,7 @@ var skippedTestsForTraditionalRoutes = []string{ // tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may // have some conflicts, skipping either one will still pass normally. // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6144 - tests.GRPCExactMethodMatching.ShortName, + tests.GRPCRouteHeaderMatching.ShortName, } var skippedTestsForExpressionRoutes = []string{ diff --git a/test/integration/consumer_group_test.go b/test/integration/consumer_group_test.go index 34e3d1ece6..aa0b1d4ca0 100644 --- a/test/integration/consumer_group_test.go +++ b/test/integration/consumer_group_test.go @@ -12,6 +12,7 @@ import ( "github.com/kong/go-kong/kong" "github.com/kong/kubernetes-testing-framework/pkg/clusters" "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -37,11 +38,16 @@ func TestConsumerGroup(t *testing.T) { ctx := context.Background() ns, cleaner := helpers.Setup(ctx, t, env) - d, s, i, p := deployMinimalSvcWithKeyAuth(ctx, t, ns.Name) - cleaner.Add(d) - cleaner.Add(s) - cleaner.Add(i) - cleaner.Add(p) + // path is the basic path used for most of the test + path := "/test-consumer-group/basic" + // multiPath is the path used to test consumer group + route plugins + multiPath := "/test-consumer-group/multi" + + deployment, service, ingress, keyauthPlugin := deployMinimalSvcWithKeyAuth(ctx, t, ns.Name, path) + cleaner.Add(deployment) + cleaner.Add(service) + cleaner.Add(ingress) + cleaner.Add(keyauthPlugin) addedHeader := header{ K: "X-Test-Header", @@ -90,6 +96,15 @@ func TestConsumerGroup(t *testing.T) { ctx, t, ns.Name, "test-consumer-group-2", pluginRateLimit.Name, ) cleaner.Add(rateLimitGroup) + // 3 has consumers but no plugins + nothingGroup := configureConsumerGroupWithPlugins( + ctx, t, ns.Name, "test-consumer-group-3", + ) + cleaner.Add(nothingGroup) + addHeaderRouteGroup := configureConsumerGroupWithPlugins( + ctx, t, ns.Name, "test-consumer-group-4", pluginRespTrans.Name, + ) + cleaner.Add(addHeaderRouteGroup) rateLimitHeader := header{ K: "RateLimit-Limit", @@ -131,7 +146,7 @@ func TestConsumerGroup(t *testing.T) { t.Log("checking if consumer has plugin configured correctly based on consumer group membership") for _, consumer := range consumers { require.Eventually(t, func() bool { - req := helpers.MustHTTPRequest(t, http.MethodGet, proxyHTTPURL.Host, "/", map[string]string{ + req := helpers.MustHTTPRequest(t, http.MethodGet, proxyHTTPURL.Host, path, map[string]string{ "apikey": consumer.Name, }) resp, err := helpers.DefaultHTTPClientWithProxy(proxyHTTPURL).Do(req) @@ -159,10 +174,77 @@ func TestConsumerGroup(t *testing.T) { return true }, ingressWait, waitTick) } + + t.Log("checking plugins attached to a consumer group and route only apply when request matches both") + four, fourSecret := configureConsumerWithAPIKey(ctx, t, ns.Name, "test-consumer-4", "test-consumer-group-4") + cleaner.Add(four) + cleaner.Add(fourSecret) + + multiIngress := generators.NewIngressForService(multiPath, map[string]string{ + annotations.AnnotationPrefix + annotations.StripPathKey: "true", + annotations.AnnotationPrefix + annotations.PluginsKey: strings.Join([]string{keyauthPlugin.Name, pluginRespTrans.Name}, ","), + }, service) + multiIngress.Spec.IngressClassName = kong.String(consts.IngressClass) + multiIngress.Name = "multi" + require.NoError(t, clusters.DeployIngress(ctx, env.Cluster(), ns.Name, multiIngress)) + cleaner.Add(multiIngress) + + require.EventuallyWithT(t, func(c *assert.CollectT) { + // this should see the header, it uses a consumer in the group on the associated route + req := helpers.MustHTTPRequest(t, http.MethodGet, proxyHTTPURL.Host, multiPath, map[string]string{ + "apikey": four.Name, + }) + resp, err := helpers.DefaultHTTPClientWithProxy(proxyHTTPURL).Do(req) + if !assert.NoError(c, err) { + return + } + defer resp.Body.Close() + if !assert.Equal(c, resp.StatusCode, http.StatusOK) { + return + } + hv := resp.Header.Get(addedHeader.K) + if !assert.Equal(c, addedHeader.V, hv) { + return + } + + // this should not see the header, it uses a consumer in the group on another route + clear := helpers.MustHTTPRequest(t, http.MethodGet, proxyHTTPURL.Host, path, map[string]string{ + "apikey": four.Name, + }) + clearResp, err := helpers.DefaultHTTPClientWithProxy(proxyHTTPURL).Do(clear) + if !assert.NoError(c, err) { + return + } + defer clearResp.Body.Close() + if !assert.Equal(c, clearResp.StatusCode, http.StatusOK) { + return + } + hv = clearResp.Header.Get(addedHeader.K) + if !assert.NotEqual(c, addedHeader.V, hv) { + return + } + + // this should not see the header, it uses a consumer outside the group on the associated route + empty := helpers.MustHTTPRequest(t, http.MethodGet, proxyHTTPURL.Host, multiPath, map[string]string{ + "apikey": "test-consumer-3", + }) + emptyResp, err := helpers.DefaultHTTPClientWithProxy(proxyHTTPURL).Do(empty) + if !assert.NoError(c, err) { + return + } + defer emptyResp.Body.Close() + if !assert.Equal(c, emptyResp.StatusCode, http.StatusOK) { + return + } + hv = emptyResp.Header.Get(addedHeader.K) + if !assert.NotEqual(c, addedHeader.V, hv) { + return + } + }, ingressWait, waitTick) } func deployMinimalSvcWithKeyAuth( - ctx context.Context, t *testing.T, namespace string, + ctx context.Context, t *testing.T, namespace, path string, ) (*appsv1.Deployment, *corev1.Service, *netv1.Ingress, *kongv1.KongPlugin) { const pluginKeyAuthName = "key-auth" t.Logf("configuring plugin %q (to give consumers an identity)", pluginKeyAuthName) @@ -195,7 +277,7 @@ func deployMinimalSvcWithKeyAuth( require.NoError(t, err) t.Logf("creating an ingress for service %q with plugin %q attached", service.Name, pluginKeyAuthName) - ingress := generators.NewIngressForService("/", map[string]string{ + ingress := generators.NewIngressForService(path, map[string]string{ annotations.AnnotationPrefix + annotations.StripPathKey: "true", annotations.AnnotationPrefix + annotations.PluginsKey: pluginKeyAuthName, }, service) From 89782def7bd5f632bba9932f157629dd23f20349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Fri, 7 Jun 2024 13:21:56 +0200 Subject: [PATCH 48/48] chore(tests): add fallback golden tests for plugins (#6161) --- .../default_golden.yaml | 3 + .../in.yaml | 197 +++++++++++++++++ .../default_golden.yaml | 3 + .../fallback-config-kong-plugin/in.yaml | 199 ++++++++++++++++++ 4 files changed, 402 insertions(+) create mode 100644 internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/default_golden.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/in.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-kong-plugin/default_golden.yaml create mode 100644 internal/dataplane/testdata/golden/fallback-config-kong-plugin/in.yaml diff --git a/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/default_golden.yaml b/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/default_golden.yaml new file mode 100644 index 0000000000..017467254d --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/default_golden.yaml @@ -0,0 +1,3 @@ +_format_version: "3.0" +upstreams: +- name: kong diff --git a/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/in.yaml b/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/in.yaml new file mode 100644 index 0000000000..db8126b1d5 --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-kong-cluster-plugin/in.yaml @@ -0,0 +1,197 @@ +# In this test case we have a set of broken KongClusterPlugins attached to all possible KongClusterPlugin's dependants. +# We expect empty config because of the broken plugins affecting all of its dependants. +# `test.konghq.com/broken` annotations can be removed from the plugins to generate the actual config. +--- +apiVersion: configuration.konghq.com/v1 +kind: KongClusterPlugin +metadata: + name: plugin + uid: "6faea5d6-ce95-439e-b223-421a0a142e3f" + annotations: + test.konghq.com/broken: "true" +config: + header_name: kong-id +plugin: correlation-id +--- +apiVersion: configuration.konghq.com/v1 +kind: KongClusterPlugin +metadata: + name: plugin-consumer-group + uid: "439e6c3b-08e7-49ff-abc9-d17a00b06ed8" + annotations: + test.konghq.com/broken: "true" +config: + header_name: kong-id +plugin: correlation-id +--- +apiVersion: v1 +kind: Service +metadata: + name: service + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + ports: + - port: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: service + port: + number: 80 + path: /ingress-service + pathType: Exact +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute + namespace: default + annotations: + konghq.com/strip-path: "true" + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - matches: + - path: + type: PathPrefix + value: /httproute + backendRefs: + - name: service + kind: Service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: tcproute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: UDPRoute +metadata: + name: udproute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: tlsroute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + hostnames: + - tlsroute.kong.example + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpcroute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + hostnames: + - "example.com" + rules: + - backendRefs: + - name: service + port: 80 + matches: + - method: + service: "grpcbin.GRPCBin" + method: "DummyUnary" +--- +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: consumer + namespace: default + annotations: + konghq.com/plugins: plugin + kubernetes.io/ingress.class: kong +username: consumer +consumerGroups: + - consumer-group +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: KongConsumerGroup +metadata: + name: consumer-group + namespace: default + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: plugin-consumer-group +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: UDPIngress +metadata: + name: udpingress + namespace: default + annotations: + kubernetes.io/ingress.class: "kong" + konghq.com/plugins: plugin +spec: + rules: + - backend: + serviceName: service + servicePort: 80 + port: 9999 +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: TCPIngress +metadata: + name: tcpingress + namespace: default + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: plugin +spec: + rules: + - port: 9999 + backend: + serviceName: service + servicePort: 80 diff --git a/internal/dataplane/testdata/golden/fallback-config-kong-plugin/default_golden.yaml b/internal/dataplane/testdata/golden/fallback-config-kong-plugin/default_golden.yaml new file mode 100644 index 0000000000..017467254d --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-kong-plugin/default_golden.yaml @@ -0,0 +1,3 @@ +_format_version: "3.0" +upstreams: +- name: kong diff --git a/internal/dataplane/testdata/golden/fallback-config-kong-plugin/in.yaml b/internal/dataplane/testdata/golden/fallback-config-kong-plugin/in.yaml new file mode 100644 index 0000000000..af10e30da5 --- /dev/null +++ b/internal/dataplane/testdata/golden/fallback-config-kong-plugin/in.yaml @@ -0,0 +1,199 @@ +# In this test case we have a set of broken KongPlugins attached to all possible KongPlugin's dependants. +# We expect empty config because of the broken plugins affecting all of its dependants. +# `test.konghq.com/broken` annotations can be removed from the plugins to generate the actual config. +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: plugin + namespace: default + uid: "6faea5d6-ce95-439e-b223-421a0a142e3f" + annotations: + test.konghq.com/broken: "true" +config: + header_name: kong-id +plugin: correlation-id +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: plugin-consumer-group + namespace: default + uid: "439e6c3b-08e7-49ff-abc9-d17a00b06ed8" + annotations: + test.konghq.com/broken: "true" +config: + header_name: kong-id +plugin: correlation-id +--- +apiVersion: v1 +kind: Service +metadata: + name: service + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + ports: + - port: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + ingressClassName: kong + rules: + - host: example.com + http: + paths: + - backend: + service: + name: service + port: + number: 80 + path: /ingress-service + pathType: Exact +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute + namespace: default + annotations: + konghq.com/strip-path: "true" + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - matches: + - path: + type: PathPrefix + value: /httproute + backendRefs: + - name: service + kind: Service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: tcproute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: UDPRoute +metadata: + name: udproute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: tlsroute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + hostnames: + - tlsroute.kong.example + rules: + - backendRefs: + - name: service + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpcroute + namespace: default + annotations: + konghq.com/plugins: plugin +spec: + parentRefs: + - name: kong + hostnames: + - "example.com" + rules: + - backendRefs: + - name: service + port: 80 + matches: + - method: + service: "grpcbin.GRPCBin" + method: "DummyUnary" +--- +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: consumer + namespace: default + annotations: + konghq.com/plugins: plugin + kubernetes.io/ingress.class: kong +username: consumer +consumerGroups: + - consumer-group +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: KongConsumerGroup +metadata: + name: consumer-group + namespace: default + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: plugin-consumer-group +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: UDPIngress +metadata: + name: udpingress + namespace: default + annotations: + kubernetes.io/ingress.class: "kong" + konghq.com/plugins: plugin +spec: + rules: + - backend: + serviceName: service + servicePort: 80 + port: 9999 +--- +apiVersion: configuration.konghq.com/v1beta1 +kind: TCPIngress +metadata: + name: tcpingress + namespace: default + annotations: + kubernetes.io/ingress.class: kong + konghq.com/plugins: plugin +spec: + rules: + - port: 9999 + backend: + serviceName: service + servicePort: 80