Skip to content

Commit 20e950e

Browse files
zreigzzreigz
authored
feat: detect usage of deprecated custom resources in our agent (#418)
* detect usage of deprecated custom resources in our agent * bump client * gen mock * find deprecated object * linter * linter * add unit test * check served flag * resolve conflicts
1 parent b84fc7a commit 20e950e

File tree

12 files changed

+934
-37
lines changed

12 files changed

+934
-37
lines changed

‎cmd/agent/console.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func registerConsoleReconcilersOrDie(
5454
consoleClient client.Client,
5555
) {
5656
mgr.AddReconcilerOrDie(service.Identifier, func() (v1.Reconciler, error) {
57-
r, err := service.NewServiceReconciler(consoleClient, config, args.ControllerCacheTTL(), args.ManifestCacheTTL(), args.ManifestCacheJitter(), args.RestoreNamespace(), args.ConsoleUrl())
57+
r, err := service.NewServiceReconciler(consoleClient, k8sClient, config, args.ControllerCacheTTL(), args.ManifestCacheTTL(), args.ManifestCacheJitter(), args.RestoreNamespace(), args.ConsoleUrl())
5858
return r, err
5959
})
6060

‎pkg/client/cluster.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package client
33
import (
44
console "github.com/pluralsh/console/go/client"
55
"github.com/pluralsh/polly/containers"
6+
"github.com/samber/lo"
67
"k8s.io/apimachinery/pkg/api/errors"
78
"k8s.io/apimachinery/pkg/runtime/schema"
89

@@ -61,7 +62,7 @@ func appendUniqueExternalDNSNamespace(slice []*string, newValue *string) []*stri
6162
return sliceSet.List()
6263
}
6364

64-
func (c *client) RegisterRuntimeServices(svcs map[string]*NamespaceVersion, serviceId *string, serviceMesh *console.ServiceMesh) error {
65+
func (c *client) RegisterRuntimeServices(svcs map[string]*NamespaceVersion, deprecated []console.DeprecatedCustomResourceAttributes, serviceId *string, serviceMesh *console.ServiceMesh) error {
6566
inputs := make([]*console.RuntimeServiceAttributes, 0)
6667
var layouts *console.OperationalLayoutAttributes
6768
for name, nv := range svcs {
@@ -96,7 +97,7 @@ func (c *client) RegisterRuntimeServices(svcs map[string]*NamespaceVersion, serv
9697
}
9798

9899
layouts = initServiceMesh(layouts, serviceMesh)
99-
_, err := c.consoleClient.RegisterRuntimeServices(c.ctx, inputs, layouts, nil, serviceId)
100+
_, err := c.consoleClient.RegisterRuntimeServices(c.ctx, inputs, layouts, lo.ToSlicePtr(deprecated), serviceId)
100101
return err
101102
}
102103

‎pkg/client/console.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ type Client interface {
4343
GetCredentials() (url, token string)
4444
PingCluster(attributes console.ClusterPing) error
4545
Ping(vsn string) error
46-
RegisterRuntimeServices(svcs map[string]*NamespaceVersion, serviceId *string, serviceMesh *console.ServiceMesh) error
46+
RegisterRuntimeServices(svcs map[string]*NamespaceVersion, deprecated []console.DeprecatedCustomResourceAttributes, serviceId *string, serviceMesh *console.ServiceMesh) error
4747
UpsertVirtualCluster(parentID string, attributes console.ClusterAttributes) (*console.GetClusterWithToken_Cluster, error)
4848
IsClusterExists(id string) (bool, error)
4949
GetCluster(id string) (*console.TinyClusterFragment, error)

‎pkg/controller/service/reconciler.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"sigs.k8s.io/cli-utils/pkg/common"
2222
"sigs.k8s.io/cli-utils/pkg/inventory"
2323
"sigs.k8s.io/cli-utils/pkg/object"
24+
ctrclient "sigs.k8s.io/controller-runtime/pkg/client"
2425
"sigs.k8s.io/controller-runtime/pkg/log"
2526
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2627

@@ -37,6 +38,7 @@ import (
3738
manis "github.com/pluralsh/deployment-operator/pkg/manifests"
3839
"github.com/pluralsh/deployment-operator/pkg/ping"
3940
"github.com/pluralsh/deployment-operator/pkg/websocket"
41+
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
4042
)
4143

4244
const (
@@ -60,9 +62,11 @@ type ServiceReconciler struct {
6062
restoreNamespace string
6163
mapper meta.RESTMapper
6264
pinger *ping.Pinger
65+
apiExtClient *apiextensionsclient.Clientset
66+
k8sClient ctrclient.Client
6367
}
6468

65-
func NewServiceReconciler(consoleClient client.Client, config *rest.Config, refresh, manifestTTL, manifestTTLJitter time.Duration, restoreNamespace, consoleURL string) (*ServiceReconciler, error) {
69+
func NewServiceReconciler(consoleClient client.Client, k8sClient ctrclient.Client, config *rest.Config, refresh, manifestTTL, manifestTTLJitter time.Duration, restoreNamespace, consoleURL string) (*ServiceReconciler, error) {
6670
utils.DisableClientLimits(config)
6771

6872
_, deployToken := consoleClient.GetCredentials()
@@ -80,7 +84,10 @@ func NewServiceReconciler(consoleClient client.Client, config *rest.Config, refr
8084
if err != nil {
8185
return nil, err
8286
}
83-
87+
apiExtClient, err := apiextensionsclient.NewForConfig(config)
88+
if err != nil {
89+
return nil, err
90+
}
8491
invFactory := inventory.ClusterClientFactory{StatusPolicy: inventory.StatusPolicyNone}
8592

8693
a, err := newApplier(invFactory, f)
@@ -109,6 +116,8 @@ func NewServiceReconciler(consoleClient client.Client, config *rest.Config, refr
109116
pinger: ping.New(consoleClient, discoveryClient, f),
110117
restoreNamespace: restoreNamespace,
111118
mapper: mapper,
119+
apiExtClient: apiExtClient,
120+
k8sClient: k8sClient,
112121
}, nil
113122
}
114123

@@ -273,14 +282,16 @@ func (s *ServiceReconciler) Poll(ctx context.Context) error {
273282
}
274283

275284
pager := s.ListServices(ctx)
285+
276286
for pager.HasNext() {
277287
services, err := pager.NextPage()
278288
if err != nil {
279289
logger.Error(err, "failed to fetch service list from deployments service")
280290
return err
281291
}
282292
for _, svc := range services {
283-
// If services arg is provided, we can skip services that are not on the list.
293+
// If services arg is provided, we can skip
294+
// services that are not on the list.
284295
if args.SkipService(svc.Node.ID) {
285296
continue
286297
}

‎pkg/controller/service/reconciler_scraper.go‎

Lines changed: 117 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,17 @@ package service
22

33
import (
44
"context"
5+
"fmt"
6+
"sort"
57
"strings"
68

79
"github.com/Masterminds/semver/v3"
10+
console "github.com/pluralsh/console/go/client"
11+
v1 "github.com/pluralsh/deployment-operator/pkg/controller/v1"
12+
"github.com/pluralsh/deployment-operator/pkg/scraper"
13+
"github.com/samber/lo"
814
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/runtime/schema"
916
"sigs.k8s.io/controller-runtime/pkg/log"
1017

1118
"github.com/pluralsh/deployment-operator/pkg/cache"
@@ -47,8 +54,7 @@ func (s *ServiceReconciler) ScrapeKube(ctx context.Context) {
4754

4855
serviceMesh := cache.ServiceMesh(hasEBPFDaemonSet)
4956
logger.Info("detected service mesh", "serviceMesh", serviceMesh)
50-
51-
if err := s.consoleClient.RegisterRuntimeServices(runtimeServices, nil, serviceMesh); err != nil {
57+
if err := s.consoleClient.RegisterRuntimeServices(runtimeServices, s.GetDeprecatedCustomResources(ctx), nil, serviceMesh); err != nil {
5258
logger.Error(err, "failed to register runtime services, this is an ignorable error but could mean your console needs to be upgraded")
5359
}
5460
}
@@ -100,3 +106,112 @@ func addVersion(services map[string]*client.NamespaceVersion, name, vsn string)
100106
services[name].Version = vsn
101107
}
102108
}
109+
110+
func (s *ServiceReconciler) getVersionedCrd(ctx context.Context) (map[string][]v1.NormalizedVersion, error) {
111+
crdList, err := s.apiExtClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
112+
if err != nil {
113+
return nil, err
114+
}
115+
crdVersionsMap := make(map[string][]v1.NormalizedVersion, len(crdList.Items))
116+
for _, crd := range crdList.Items {
117+
kind := crd.Spec.Names.Kind
118+
group := crd.Spec.Group
119+
groupKind := fmt.Sprintf("%s/%s", group, kind)
120+
var parsedVersions []v1.NormalizedVersion
121+
for _, v := range crd.Spec.Versions {
122+
parsed, ok := v1.ParseVersion(v.Name)
123+
if !ok {
124+
continue
125+
}
126+
// flag enabling/disabling this version from being served via REST APIs
127+
if !v.Served {
128+
continue
129+
}
130+
parsedVersions = append(parsedVersions, *parsed)
131+
}
132+
sort.Slice(parsedVersions, func(i, j int) bool {
133+
return v1.CompareVersions(parsedVersions[i], parsedVersions[j])
134+
})
135+
crdVersionsMap[groupKind] = parsedVersions
136+
}
137+
138+
return crdVersionsMap, nil
139+
}
140+
141+
func (s *ServiceReconciler) GetDeprecatedCustomResources(ctx context.Context) []console.DeprecatedCustomResourceAttributes {
142+
logger := log.FromContext(ctx)
143+
crds, err := s.getVersionedCrd(ctx)
144+
if err != nil {
145+
logger.Error(err, "failed to retrieve versioned CRDs")
146+
return nil
147+
}
148+
149+
var deprecated []console.DeprecatedCustomResourceAttributes
150+
for groupKind, versions := range crds {
151+
gkList := strings.Split(groupKind, "/")
152+
if len(gkList) != 2 {
153+
continue
154+
}
155+
group := gkList[0]
156+
kind := gkList[1]
157+
d := s.getDeprecatedCustomResourceObjects(ctx, versions, group, kind)
158+
deprecated = append(deprecated, d...)
159+
}
160+
return deprecated
161+
}
162+
163+
func (s *ServiceReconciler) getDeprecatedCustomResourceObjects(ctx context.Context, versions []v1.NormalizedVersion, group, kind string) []console.DeprecatedCustomResourceAttributes {
164+
var deprecatedCustomResourceAttributes []console.DeprecatedCustomResourceAttributes
165+
versionPairs := getVersionPairs(versions)
166+
for _, version := range versionPairs {
167+
gvk := schema.GroupVersionKind{
168+
Group: group,
169+
Version: version.PreviousVersion,
170+
Kind: kind,
171+
}
172+
173+
pager := scraper.ListResources(ctx, s.k8sClient, gvk, nil)
174+
for pager.HasNext() {
175+
items, err := pager.NextPage()
176+
if err != nil {
177+
break
178+
}
179+
for _, item := range items {
180+
attr := console.DeprecatedCustomResourceAttributes{
181+
Group: group,
182+
Kind: kind,
183+
Name: item.GetName(),
184+
Version: version.PreviousVersion,
185+
NextVersion: version.LatestVersion,
186+
}
187+
if item.GetNamespace() != "" {
188+
attr.Namespace = lo.ToPtr(item.GetNamespace())
189+
}
190+
deprecatedCustomResourceAttributes = append(deprecatedCustomResourceAttributes, attr)
191+
}
192+
}
193+
}
194+
return deprecatedCustomResourceAttributes
195+
}
196+
197+
type VersionPair struct {
198+
LatestVersion string
199+
PreviousVersion string
200+
}
201+
202+
func getVersionPairs(versions []v1.NormalizedVersion) []VersionPair {
203+
// Helper function for creating VersionPair
204+
createVersionPair := func(latest, previous v1.NormalizedVersion) VersionPair {
205+
return VersionPair{
206+
LatestVersion: latest.Raw,
207+
PreviousVersion: previous.Raw,
208+
}
209+
}
210+
211+
versionPairs := make([]VersionPair, 0, len(versions)-1) // Preallocate slice capacity
212+
for i := 0; i < len(versions)-1; i++ {
213+
versionPair := createVersionPair(versions[i], versions[i+1])
214+
versionPairs = append(versionPairs, versionPair)
215+
}
216+
return versionPairs
217+
}

‎pkg/controller/service/reconciler_scraper_test.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package service_test
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/pluralsh/deployment-operator/pkg/controller/service"
8+
"github.com/pluralsh/deployment-operator/pkg/test/mocks"
9+
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
10+
"k8s.io/apimachinery/pkg/api/errors"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/types"
13+
14+
. "github.com/onsi/ginkgo/v2"
15+
. "github.com/onsi/gomega"
16+
)
17+
18+
var _ = Describe("Scraper", Ordered, func() {
19+
Context("When reconciling a resource", func() {
20+
const (
21+
resourceName = "default"
22+
namespace = "default"
23+
)
24+
25+
ctx := context.Background()
26+
27+
typeNamespacedName := types.NamespacedName{
28+
Name: resourceName,
29+
Namespace: "default",
30+
}
31+
32+
backup := &velerov1.Backup{}
33+
34+
BeforeAll(func() {
35+
By("creating the custom resource for the Kind Backup")
36+
err := kClient.Get(ctx, typeNamespacedName, backup)
37+
if err != nil && errors.IsNotFound(err) {
38+
resource := &velerov1.Backup{
39+
ObjectMeta: metav1.ObjectMeta{
40+
Name: resourceName,
41+
Namespace: namespace,
42+
},
43+
Spec: velerov1.BackupSpec{},
44+
}
45+
Expect(kClient.Create(ctx, resource)).To(Succeed())
46+
}
47+
})
48+
49+
AfterAll(func() {
50+
resource := &velerov1.Backup{}
51+
err := kClient.Get(ctx, typeNamespacedName, resource)
52+
Expect(err).NotTo(HaveOccurred())
53+
54+
By("Cleanup the specific resource instance Backup")
55+
Expect(kClient.Delete(ctx, resource)).To(Succeed())
56+
})
57+
58+
It("should return deprecated resources", func() {
59+
fakeConsoleClient := mocks.NewClientMock(mocks.TestingT)
60+
fakeConsoleClient.On("GetCredentials").Return("", "")
61+
62+
reconciler, err := service.NewServiceReconciler(fakeConsoleClient, kClient, cfg, time.Minute, time.Minute, time.Minute, namespace, "http://localhost:8080")
63+
Expect(err).NotTo(HaveOccurred())
64+
ds := reconciler.GetDeprecatedCustomResources(ctx)
65+
Expect(ds).To(HaveLen(1))
66+
Expect(ds[0].Version).To(Equal("v1"))
67+
Expect(ds[0].NextVersion).To(Equal("v2alpha1"))
68+
})
69+
})
70+
})

‎pkg/controller/service/reconciler_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ var _ = Describe("Reconciler", Ordered, func() {
107107
fakeConsoleClient.On("UpdateComponents", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
108108
fakeConsoleClient.On("UpdateServiceErrors", mock.Anything, mock.Anything).Return(nil)
109109

110-
reconciler, err := service.NewServiceReconciler(fakeConsoleClient, cfg, time.Minute, time.Minute, time.Second*10, namespace, "http://localhost:8080")
110+
reconciler, err := service.NewServiceReconciler(fakeConsoleClient, kClient, cfg, time.Minute, time.Minute, time.Second*10, namespace, "http://localhost:8080")
111111
Expect(err).NotTo(HaveOccurred())
112112
_, err = reconciler.Reconcile(ctx, serviceId)
113113
Expect(err).NotTo(HaveOccurred())

‎pkg/controller/service/suite_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,11 @@ import (
2222
"runtime"
2323
"testing"
2424

25-
"k8s.io/client-go/rest"
26-
27-
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
28-
2925
. "github.com/onsi/ginkgo/v2"
3026
. "github.com/onsi/gomega"
27+
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
3128
"k8s.io/client-go/kubernetes/scheme"
29+
"k8s.io/client-go/rest"
3230
"sigs.k8s.io/controller-runtime/pkg/client"
3331
"sigs.k8s.io/controller-runtime/pkg/envtest"
3432
logf "sigs.k8s.io/controller-runtime/pkg/log"

0 commit comments

Comments
 (0)