Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: improve VPA filtering and checkpoint garbage collection #7716

Merged
merged 6 commits into from
Feb 5, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions vertical-pod-autoscaler/common/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func InitCommonFlags() *CommonFlags {
flag.Float64Var(&cf.KubeApiQps, "kube-api-qps", 5.0, "QPS limit when making requests to Kubernetes apiserver")
flag.Float64Var(&cf.KubeApiBurst, "kube-api-burst", 10.0, "QPS burst limit when making requests to Kubernetes apiserver")
flag.BoolVar(&cf.EnableProfiling, "profiling", false, "Is debug/pprof endpoint enabled")
flag.StringVar(&cf.VpaObjectNamespace, "vpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for VPA objects. Empty means all namespaces will be used.")
flag.StringVar(&cf.IgnoredVpaObjectNamespaces, "ignored-vpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for VPA objects. Empty means no namespaces will be ignored.")
flag.StringVar(&cf.VpaObjectNamespace, "vpa-object-namespace", apiv1.NamespaceAll, "Specifies the namespace to search for VPA objects. Leave empty to include all namespaces. If provided, the garbage collector will only clean this namespace.")
flag.StringVar(&cf.IgnoredVpaObjectNamespaces, "ignored-vpa-object-namespaces", "", "A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector.")
return cf
}

Expand Down
56 changes: 43 additions & 13 deletions vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"slices"
"time"

"github.com/pkg/errors"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
Expand Down Expand Up @@ -89,6 +90,7 @@ type ClusterStateFeederFactory struct {
ControllerFetcher controllerfetcher.ControllerFetcher
RecommenderName string
IgnoredNamespaces []string
VpaObjectNamespace string
}

// Make creates new ClusterStateFeeder with internal data providers, based on kube client.
Expand All @@ -106,6 +108,7 @@ func (m ClusterStateFeederFactory) Make() *clusterStateFeeder {
controllerFetcher: m.ControllerFetcher,
recommenderName: m.RecommenderName,
ignoredNamespaces: m.IgnoredNamespaces,
vpaObjectNamespace: m.VpaObjectNamespace,
}
}

Expand Down Expand Up @@ -198,6 +201,7 @@ type clusterStateFeeder struct {
controllerFetcher controllerfetcher.ControllerFetcher
recommenderName string
ignoredNamespaces []string
vpaObjectNamespace string
}

func (feeder *clusterStateFeeder) InitFromHistoryProvider(historyProvider history.HistoryProvider) {
Expand Down Expand Up @@ -300,24 +304,50 @@ func (feeder *clusterStateFeeder) GarbageCollectCheckpoints() {

for _, namespaceItem := range namespaceList.Items {
namespace := namespaceItem.Name
checkpointList, err := feeder.vpaCheckpointClient.VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), metav1.ListOptions{})
// Clean the namespace if any of the following conditions are true:
// 1. `vpaObjectNamespace` is set and matches the current namespace.
// 2. `ignoredNamespaces` is set, but the current namespace is not in the list.
// 3. Neither `vpaObjectNamespace` nor `ignoredNamespaces` is set, so all namespaces are included.
if feeder.shouldIgnoreNamespace(namespace) {
klog.V(3).InfoS("Skipping namespace; it does not meet cleanup criteria", "namespace", namespace, "vpaObjectNamespace", feeder.vpaObjectNamespace, "ignoredNamespaces", feeder.ignoredNamespaces)
continue
}
err := feeder.cleanupCheckpointsForNamespace(namespace, allVPAKeys)
if err != nil {
klog.ErrorS(err, "Cannot list VPA checkpoints", "namespace", namespace)
klog.ErrorS(err, "error cleanining checkpoints")
}
for _, checkpoint := range checkpointList.Items {
vpaID := model.VpaID{Namespace: checkpoint.Namespace, VpaName: checkpoint.Spec.VPAObjectName}
exists := allVPAKeys[vpaID]
}
}

if !exists {
err = feeder.vpaCheckpointClient.VerticalPodAutoscalerCheckpoints(namespace).Delete(context.TODO(), checkpoint.Name, metav1.DeleteOptions{})
if err == nil {
klog.V(3).InfoS("Orphaned VPA checkpoint cleanup - deleting", "checkpoint", klog.KRef(namespace, checkpoint.Name))
} else {
klog.ErrorS(err, "Orphaned VPA checkpoint cleanup - error deleting", "checkpoint", klog.KRef(namespace, checkpoint.Name))
}
func (feeder *clusterStateFeeder) shouldIgnoreNamespace(namespace string) bool {
// 1. `vpaObjectNamespace` is set but doesn't match the current namespace.
if feeder.vpaObjectNamespace != "" && namespace != feeder.vpaObjectNamespace {
return true
}
// 2. `ignoredNamespaces` is set, and the current namespace is in the list.
if len(feeder.ignoredNamespaces) > 0 && slices.Contains(feeder.ignoredNamespaces, namespace) {
return true
}
return false
}

func (feeder *clusterStateFeeder) cleanupCheckpointsForNamespace(namespace string, allVPAKeys map[model.VpaID]bool) error {
var err error
checkpointList, err := feeder.vpaCheckpointClient.VerticalPodAutoscalerCheckpoints(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return err
}
for _, checkpoint := range checkpointList.Items {
vpaID := model.VpaID{Namespace: checkpoint.Namespace, VpaName: checkpoint.Spec.VPAObjectName}
if !allVPAKeys[vpaID] {
if errFeeder := feeder.vpaCheckpointClient.VerticalPodAutoscalerCheckpoints(namespace).Delete(context.TODO(), checkpoint.Name, metav1.DeleteOptions{}); errFeeder != nil {
err = errors.Wrapf(err, "failed to delete orphaned checkpoint %s", klog.KRef(namespace, checkpoint.Name))
continue
}
klog.V(3).InfoS("Orphaned VPA checkpoint cleanup - deleting", "checkpoint", klog.KRef(namespace, checkpoint.Name))
}
}
return err
}

func implicitDefaultRecommender(selectors []*vpa_types.VerticalPodAutoscalerRecommenderSelector) bool {
Expand Down Expand Up @@ -354,7 +384,7 @@ func filterVPAs(feeder *clusterStateFeeder, allVpaCRDs []*vpa_types.VerticalPodA
}
}

if slices.Contains(feeder.ignoredNamespaces, vpaCRD.ObjectMeta.Namespace) {
if feeder.shouldIgnoreNamespace(vpaCRD.ObjectMeta.Namespace) {
klog.V(6).InfoS("Ignoring vpaCRD as this namespace is ignored", "vpaCRD", klog.KObj(vpaCRD))
continue
}
Expand Down
1 change: 1 addition & 0 deletions vertical-pod-autoscaler/pkg/recommender/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ func run(healthCheck *metrics.HealthCheck, commonFlag *common.CommonFlags) {
ControllerFetcher: controllerFetcher,
RecommenderName: *recommenderName,
IgnoredNamespaces: ignoredNamespaces,
VpaObjectNamespace: commonFlag.VpaObjectNamespace,
}.Make()
controllerFetcher.Start(context.Background(), scaleCacheLoopPeriod)

Expand Down
Loading