From 3e5d9af71cc4e414caa316a35862a14eb298ff10 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Mon, 28 Oct 2024 15:17:17 +0300 Subject: [PATCH 1/5] feat: cron jobs config model Signed-off-by: Vladislav Sukhin --- api/testworkflows/v1/types.go | 2 + api/testworkflows/v1/zz_generated.deepcopy.go | 46 ++++++++++++++++++- ...stworkflows.testkube.io_testworkflows.yaml | 8 ++++ ...ows.testkube.io_testworkflowtemplates.yaml | 8 ++++ .../testworkflows/testworkflow_controller.go | 11 ++++- 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/api/testworkflows/v1/types.go b/api/testworkflows/v1/types.go index 5d43dbe8..a7ec1ba3 100644 --- a/api/testworkflows/v1/types.go +++ b/api/testworkflows/v1/types.go @@ -222,6 +222,8 @@ type CronJobConfig struct { Labels map[string]string `json:"labels,omitempty" expr:"template,template"` // annotations to attach to the cron job Annotations map[string]string `json:"annotations,omitempty" expr:"template,template"` + // configuration to pass for the workflow + Config map[string]intstr.IntOrString `json:"config,omitempty" expr:"template,template"` } type TestWorkflowTagSchema struct { diff --git a/api/testworkflows/v1/zz_generated.deepcopy.go b/api/testworkflows/v1/zz_generated.deepcopy.go index cb445387..2b0cfc95 100644 --- a/api/testworkflows/v1/zz_generated.deepcopy.go +++ b/api/testworkflows/v1/zz_generated.deepcopy.go @@ -242,6 +242,13 @@ func (in *CronJobConfig) DeepCopyInto(out *CronJobConfig) { (*out)[key] = val } } + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = make(map[string]intstr.IntOrString, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CronJobConfig. @@ -321,7 +328,7 @@ func (in *IndependentServiceSpec) DeepCopy() *IndependentServiceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IndependentStep) DeepCopyInto(out *IndependentStep) { *out = *in - out.StepMeta = in.StepMeta + in.StepMeta.DeepCopyInto(&out.StepMeta) in.StepControl.DeepCopyInto(&out.StepControl) in.StepSource.DeepCopyInto(&out.StepSource) if in.Services != nil { @@ -729,7 +736,7 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Step) DeepCopyInto(out *Step) { *out = *in - out.StepMeta = in.StepMeta + in.StepMeta.DeepCopyInto(&out.StepMeta) in.StepControl.DeepCopyInto(&out.StepControl) if in.Use != nil { in, out := &in.Use, &out.Use @@ -988,6 +995,11 @@ func (in *StepExecuteWorkflow) DeepCopy() *StepExecuteWorkflow { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StepMeta) DeepCopyInto(out *StepMeta) { *out = *in + if in.Pure != nil { + in, out := &in.Pure, &out.Pure + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StepMeta. @@ -1775,6 +1787,11 @@ func (in *TestWorkflowSpecBase) DeepCopyInto(out *TestWorkflowSpecBase) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.System != nil { + in, out := &in.System, &out.System + *out = new(TestWorkflowSystem) + (*in).DeepCopyInto(*out) + } if in.Config != nil { in, out := &in.Config, &out.Config *out = make(map[string]ParameterSchema, len(*in)) @@ -1896,6 +1913,31 @@ func (in *TestWorkflowSummary) DeepCopy() *TestWorkflowSummary { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestWorkflowSystem) DeepCopyInto(out *TestWorkflowSystem) { + *out = *in + if in.PureByDefault != nil { + in, out := &in.PureByDefault, &out.PureByDefault + *out = new(bool) + **out = **in + } + if in.IsolatedContainers != nil { + in, out := &in.IsolatedContainers, &out.IsolatedContainers + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestWorkflowSystem. +func (in *TestWorkflowSystem) DeepCopy() *TestWorkflowSystem { + if in == nil { + return nil + } + out := new(TestWorkflowSystem) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestWorkflowTagSchema) DeepCopyInto(out *TestWorkflowTagSchema) { *out = *in diff --git a/config/crd/bases/testworkflows.testkube.io_testworkflows.yaml b/config/crd/bases/testworkflows.testkube.io_testworkflows.yaml index 57513e5b..34eaa15e 100644 --- a/config/crd/bases/testworkflows.testkube.io_testworkflows.yaml +++ b/config/crd/bases/testworkflows.testkube.io_testworkflows.yaml @@ -3581,6 +3581,14 @@ spec: type: string description: annotations to attach to the cron job type: object + config: + additionalProperties: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + description: configuration to pass for the workflow + type: object cron: description: cron schedule to run a test workflow type: string diff --git a/config/crd/bases/testworkflows.testkube.io_testworkflowtemplates.yaml b/config/crd/bases/testworkflows.testkube.io_testworkflowtemplates.yaml index c6d72f67..f300bfdf 100644 --- a/config/crd/bases/testworkflows.testkube.io_testworkflowtemplates.yaml +++ b/config/crd/bases/testworkflows.testkube.io_testworkflowtemplates.yaml @@ -3507,6 +3507,14 @@ spec: type: string description: annotations to attach to the cron job type: object + config: + additionalProperties: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + description: configuration to pass for the workflow + type: object cron: description: cron schedule to run a test workflow type: string diff --git a/internal/controller/testworkflows/testworkflow_controller.go b/internal/controller/testworkflows/testworkflow_controller.go index fac25cee..97c75790 100644 --- a/internal/controller/testworkflows/testworkflow_controller.go +++ b/internal/controller/testworkflows/testworkflow_controller.go @@ -18,6 +18,7 @@ package testworkflows import ( "context" + "encoding/json" "fmt" "maps" "net/http" @@ -128,6 +129,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request Cron: event.Cronjob.Cron, Labels: event.Cronjob.Labels, Annotations: event.Cronjob.Annotations, + Config: event.Cronjob.Config, } } else { newCronJobConfigs[cronJob.Cron] = MergeCronJobJobConfig(cronJob, event.Cronjob) @@ -135,6 +137,11 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request } } + data, err := json.Marshal(testworkflowsv1.TestWorkflowExecutionRequest{}) + if err != nil { + return ctrl.Result{}, err + } + for schedule, oldCronJob := range oldCronJobs { if newCronJobConfig, ok := newCronJobConfigs[schedule]; !ok { // Delete removed Cron Jobs @@ -157,7 +164,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request ResourceURI: cronjob.TestWorkflowResourceURI, Labels: newCronJobConfig.Labels, Annotations: newCronJobConfig.Annotations, - Data: "{}", + Data: string(data), } if err = r.CronJobClient.Update(ctx, oldCronJob, testWorkflow.Name, @@ -184,7 +191,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request ResourceURI: cronjob.TestWorkflowResourceURI, Labels: newCronJobConfig.Labels, Annotations: newCronJobConfig.Annotations, - Data: "{}", + Data: string(data), } if err = r.CronJobClient.Create(ctx, testWorkflow.Name, From 6a236843ffca5d57d576f1a465675b076ee48e8d Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Tue, 29 Oct 2024 14:42:41 +0300 Subject: [PATCH 2/5] fix: pass config to cronjob Signed-off-by: Vladislav Sukhin --- .../testworkflows/testworkflow_controller.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/controller/testworkflows/testworkflow_controller.go b/internal/controller/testworkflows/testworkflow_controller.go index e35f7751..43211c5e 100644 --- a/internal/controller/testworkflows/testworkflow_controller.go +++ b/internal/controller/testworkflows/testworkflow_controller.go @@ -139,7 +139,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request interface_ := testworkflowsv1.API_TestWorkflowRunningContextInterfaceType actor := testworkflowsv1.CRON_TestWorkflowRunningContextActorType - data, err := json.Marshal(testworkflowsv1.TestWorkflowExecutionRequest{ + request := testworkflowsv1.TestWorkflowExecutionRequest{ RunningContext: &testworkflowsv1.TestWorkflowRunningContext{ Interface_: &testworkflowsv1.TestWorkflowRunningContextInterface{ Type_: &interface_, @@ -148,9 +148,6 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request Type_: &actor, }, }, - }) - if err != nil { - return ctrl.Result{}, err } for schedule, oldCronJob := range oldCronJobs { @@ -166,6 +163,12 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request newCronJobConfig.Labels = make(map[string]string) } + request.Config = newCronJobConfig.Config + data, err := json.Marshal(request) + if err != nil { + return ctrl.Result{}, err + } + newCronJobConfig.Labels[cronjob.TestWorkflowResourceURI] = testWorkflow.Name options := cronjob.CronJobOptions{ Schedule: schedule, @@ -193,6 +196,12 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request newCronJobConfig.Labels = make(map[string]string) } + request.Config = newCronJobConfig.Config + data, err := json.Marshal(request) + if err != nil { + return ctrl.Result{}, err + } + newCronJobConfig.Labels[cronjob.TestWorkflowResourceURI] = testWorkflow.Name options := cronjob.CronJobOptions{ Schedule: schedule, From 151ffd337c8feb91efb9e0c0fa850cd9d8ba3ec7 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Tue, 29 Oct 2024 14:49:38 +0300 Subject: [PATCH 3/5] fix: merge cronjob config Signed-off-by: Vladislav Sukhin --- .../controller/testworkflows/testworkflow_controller.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/testworkflows/testworkflow_controller.go b/internal/controller/testworkflows/testworkflow_controller.go index 43211c5e..2e393323 100644 --- a/internal/controller/testworkflows/testworkflow_controller.go +++ b/internal/controller/testworkflows/testworkflow_controller.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -242,6 +243,11 @@ func MergeCronJobJobConfig(dst, include *testworkflowsv1.CronJobConfig) *testwor } maps.Copy(dst.Annotations, include.Annotations) + if len(include.Config) > 0 && dst.Config == nil { + dst.Config = map[string]intstr.IntOrString{} + } + maps.Copy(dst.Config, include.Config) + return dst } From 988284de71c65e161b43061b9b92afa6f644d742 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Tue, 29 Oct 2024 15:49:06 +0300 Subject: [PATCH 4/5] fix: allow multiple crons for different config Signed-off-by: Vladislav Sukhin --- .../testworkflows/testworkflow_controller.go | 34 ++++++++++--------- pkg/cronjob/client.go | 16 ++++++--- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/internal/controller/testworkflows/testworkflow_controller.go b/internal/controller/testworkflows/testworkflow_controller.go index 2e393323..2dc127a7 100644 --- a/internal/controller/testworkflows/testworkflow_controller.go +++ b/internal/controller/testworkflows/testworkflow_controller.go @@ -120,20 +120,25 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request } for i := range cronJobList.Items { - oldCronJobs[cronJobList.Items[i].Spec.Schedule] = &cronJobList.Items[i] + oldCronJobs[cronJobList.Items[i].Name] = &cronJobList.Items[i] } for _, event := range events { if event.Cronjob != nil { - if cronJob, ok := newCronJobConfigs[event.Cronjob.Cron]; !ok { - newCronJobConfigs[event.Cronjob.Cron] = &testworkflowsv1.CronJobConfig{ + name, err := cronjob.GetHashedMetadataName(testWorkflow.Name, event.Cronjob.Cron, event.Cronjob.Config) + if err != nil { + return ctrl.Result{}, err + } + + if cronJob, ok := newCronJobConfigs[name]; !ok { + newCronJobConfigs[name] = &testworkflowsv1.CronJobConfig{ Cron: event.Cronjob.Cron, Labels: event.Cronjob.Labels, Annotations: event.Cronjob.Annotations, Config: event.Cronjob.Config, } } else { - newCronJobConfigs[cronJob.Cron] = MergeCronJobJobConfig(cronJob, event.Cronjob) + newCronJobConfigs[name] = MergeCronJobJobConfig(cronJob, event.Cronjob) } } } @@ -151,11 +156,10 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request }, } - for schedule, oldCronJob := range oldCronJobs { - if newCronJobConfig, ok := newCronJobConfigs[schedule]; !ok { + for name, oldCronJob := range oldCronJobs { + if newCronJobConfig, ok := newCronJobConfigs[name]; !ok { // Delete removed Cron Jobs - if err = r.CronJobClient.Delete(ctx, - cronjob.GetHashedMetadataName(testWorkflow.Name, schedule), testWorkflow.Namespace); err != nil { + if err = r.CronJobClient.Delete(ctx, name, testWorkflow.Namespace); err != nil { return ctrl.Result{}, err } } else { @@ -172,7 +176,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request newCronJobConfig.Labels[cronjob.TestWorkflowResourceURI] = testWorkflow.Name options := cronjob.CronJobOptions{ - Schedule: schedule, + Schedule: newCronJobConfig.Cron, Group: testworkflowsv1.Group, Resource: testworkflowsv1.Resource, Version: testworkflowsv1.Version, @@ -182,16 +186,15 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request Data: string(data), } - if err = r.CronJobClient.Update(ctx, oldCronJob, testWorkflow.Name, - cronjob.GetHashedMetadataName(testWorkflow.Name, schedule), testWorkflow.Namespace, + if err = r.CronJobClient.Update(ctx, oldCronJob, testWorkflow.Name, name, testWorkflow.Namespace, string(testWorkflow.UID), options); err != nil { return ctrl.Result{}, err } } } - for schedule, newCronJobConfig := range newCronJobConfigs { - if _, ok = oldCronJobs[schedule]; !ok { + for name, newCronJobConfig := range newCronJobConfigs { + if _, ok = oldCronJobs[name]; !ok { // Create new Cron Jobs if newCronJobConfig.Labels == nil { newCronJobConfig.Labels = make(map[string]string) @@ -205,7 +208,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request newCronJobConfig.Labels[cronjob.TestWorkflowResourceURI] = testWorkflow.Name options := cronjob.CronJobOptions{ - Schedule: schedule, + Schedule: newCronJobConfig.Cron, Group: testworkflowsv1.Group, Resource: testworkflowsv1.Resource, Version: testworkflowsv1.Version, @@ -215,8 +218,7 @@ func (r *TestWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request Data: string(data), } - if err = r.CronJobClient.Create(ctx, testWorkflow.Name, - cronjob.GetHashedMetadataName(testWorkflow.Name, schedule), testWorkflow.Namespace, + if err = r.CronJobClient.Create(ctx, testWorkflow.Name, name, testWorkflow.Namespace, string(testWorkflow.UID), options); err != nil { return ctrl.Result{}, err } diff --git a/pkg/cronjob/client.go b/pkg/cronjob/client.go index 3796de63..9a14de66 100644 --- a/pkg/cronjob/client.go +++ b/pkg/cronjob/client.go @@ -3,6 +3,7 @@ package cronjob import ( "bytes" "context" + "encoding/json" "fmt" "hash/fnv" "maps" @@ -14,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" kyaml "sigs.k8s.io/kustomize/kyaml/yaml" @@ -294,17 +296,23 @@ func GetMetadataName(name, resource string) string { } // GetHashedMetadataName returns cron job hashed metadata name -func GetHashedMetadataName(name, schedule string) string { - h := fnv.New32a() +func GetHashedMetadataName(name, schedule string, config map[string]intstr.IntOrString) (string, error) { + data, err := json.Marshal(config) + if err != nil { + return "", err + } + + h := fnv.New64a() h.Write([]byte(schedule)) + h.Write(data) - hash := fmt.Sprintf("-%d", h.Sum32()) + hash := fmt.Sprintf("-%d", h.Sum64()) if len(name) > 52-len(hash) { name = name[:52-len(hash)] } - return name + hash + return name + hash, nil } // GetSelector returns cron job selecttor From 0a906ecd42605238f170091eef5d3abb4cb59612 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Tue, 29 Oct 2024 15:55:25 +0300 Subject: [PATCH 5/5] fix: remove merge similar config Signed-off-by: Vladislav Sukhin --- .../controller/testworkflows/testworkflow_controller.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/controller/testworkflows/testworkflow_controller.go b/internal/controller/testworkflows/testworkflow_controller.go index 2dc127a7..01c38b3d 100644 --- a/internal/controller/testworkflows/testworkflow_controller.go +++ b/internal/controller/testworkflows/testworkflow_controller.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -245,11 +244,6 @@ func MergeCronJobJobConfig(dst, include *testworkflowsv1.CronJobConfig) *testwor } maps.Copy(dst.Annotations, include.Annotations) - if len(include.Config) > 0 && dst.Config == nil { - dst.Config = map[string]intstr.IntOrString{} - } - maps.Copy(dst.Config, include.Config) - return dst }