feat: Add a locust-loandgen chaos fault #733
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,160 @@ | ||
| package lib | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/litmuschaos/litmus-go/pkg/cerrors" | ||
| "github.com/litmuschaos/litmus-go/pkg/clients" | ||
| "github.com/litmuschaos/litmus-go/pkg/events" | ||
| experimentTypes "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/types" | ||
| "github.com/litmuschaos/litmus-go/pkg/log" | ||
| "github.com/litmuschaos/litmus-go/pkg/probe" | ||
| "github.com/litmuschaos/litmus-go/pkg/status" | ||
| "github.com/litmuschaos/litmus-go/pkg/telemetry" | ||
| "github.com/litmuschaos/litmus-go/pkg/types" | ||
| "github.com/litmuschaos/litmus-go/pkg/utils/common" | ||
| "github.com/litmuschaos/litmus-go/pkg/utils/stringutils" | ||
| "github.com/palantir/stacktrace" | ||
| "go.opentelemetry.io/otel" | ||
| corev1 "k8s.io/api/core/v1" | ||
| v1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "strconv" | ||
| ) | ||
|
|
||
| func experimentExecution(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error { | ||
| ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectLocustLoadGenFault") | ||
| defer span.End() | ||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on target pod" | ||
| types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails) | ||
| events.GenerateEvents(eventsDetails, clients, chaosDetails, "ChaosEngine") | ||
| } | ||
|
|
||
| // run the probes during chaos | ||
| if len(resultDetails.ProbeDetails) != 0 { | ||
| if err := probe.RunProbes(ctx, chaosDetails, clients, resultDetails, "DuringChaos", eventsDetails); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| runID := stringutils.GetRunID() | ||
|
|
||
| // creating the helper pod to perform locust-loadgen chaos | ||
| if err := createHelperPod(ctx, experimentsDetails, clients, chaosDetails, runID); err != nil { | ||
| return stacktrace.Propagate(err, "could not create helper pod") | ||
| } | ||
|
|
||
| appLabel := fmt.Sprintf("app=%s-helper-%s", experimentsDetails.ExperimentName, runID) | ||
|
|
||
| //checking the status of the helper pod, wait till the pod comes to running state else fail the experiment | ||
| log.Info("[Status]: Checking the status of the helper pod") | ||
| if err := status.CheckHelperStatus(experimentsDetails.ChaosNamespace, appLabel, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil { | ||
| common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients) | ||
| return stacktrace.Propagate(err, "could not check helper status") | ||
| } | ||
|
|
||
| // Wait till the completion of the helper pod | ||
| // set an upper limit for the waiting time | ||
| log.Info("[Wait]: waiting till the completion of the helper pod") | ||
| podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, appLabel, clients, experimentsDetails.ChaosDuration+experimentsDetails.Timeout, common.GetContainerNames(chaosDetails)...) | ||
| if err != nil || podStatus == "Failed" { | ||
| common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients) | ||
| return common.HelperFailedError(err, appLabel, experimentsDetails.ChaosNamespace, true) | ||
| } | ||
|
|
||
| //Deleting all the helper pod for locust-loadgen chaos | ||
| log.Info("[Cleanup]: Deleting all the helper pods") | ||
| if err = common.DeleteAllPod(appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil { | ||
| return stacktrace.Propagate(err, "could not delete helper pod(s)") | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // PrepareChaos contains the preparation steps before chaos injection | ||
| func PrepareChaos(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error { | ||
| ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareLocustLoadGenFault") | ||
| defer span.End() | ||
|
|
||
| //Waiting for the ramp time before chaos injection | ||
| if experimentsDetails.RampTime != 0 { | ||
| log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime) | ||
| common.WaitForDuration(experimentsDetails.RampTime) | ||
| } | ||
| //Starting the locust-loadgen experiment | ||
| if err := experimentExecution(ctx, experimentsDetails, clients, resultDetails, eventsDetails, chaosDetails); err != nil { | ||
| return stacktrace.Propagate(err, "could not execute chaos") | ||
| } | ||
| //Waiting for the ramp time after chaos injection | ||
| if experimentsDetails.RampTime != 0 { | ||
| log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime) | ||
| common.WaitForDuration(experimentsDetails.RampTime) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // createHelperPod derive the attributes for helper pod and create the helper pod | ||
| func createHelperPod(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, runID string) error { | ||
| ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "CreateLocustLoadGenFaultHelperPod") | ||
| defer span.End() | ||
|
|
||
| const volumeName = "script-volume" | ||
| const mountPath = "/mnt" | ||
|
|
||
| helperPod := &corev1.Pod{ | ||
| ObjectMeta: v1.ObjectMeta{ | ||
| GenerateName: experimentsDetails.ExperimentName + "-helper-", | ||
| Namespace: experimentsDetails.ChaosNamespace, | ||
| Labels: common.GetHelperLabels(chaosDetails.Labels, runID, experimentsDetails.ExperimentName), | ||
| Annotations: chaosDetails.Annotations, | ||
| }, | ||
| Spec: corev1.PodSpec{ | ||
| RestartPolicy: corev1.RestartPolicyNever, | ||
| ImagePullSecrets: chaosDetails.ImagePullSecrets, | ||
| Containers: []corev1.Container{ | ||
| { | ||
| Name: experimentsDetails.ExperimentName, | ||
| Image: experimentsDetails.LIBImage, | ||
| ImagePullPolicy: corev1.PullPolicy(experimentsDetails.LIBImagePullPolicy), | ||
| Command: []string{ | ||
| "locust", | ||
| "--headless", | ||
| }, | ||
| Args: []string{ | ||
| "--users", strconv.Itoa(experimentsDetails.Users), | ||
| "--spawn-rate", strconv.Itoa(experimentsDetails.SpawnRate), | ||
| "-H", experimentsDetails.Host, | ||
| "-t", strconv.Itoa(experimentsDetails.ChaosDuration) + "s", | ||
| "-f", mountPath + "/" + experimentsDetails.ConfigMapKey, | ||
| }, | ||
| Resources: chaosDetails.Resources, | ||
| VolumeMounts: []corev1.VolumeMount{ | ||
| { | ||
| Name: volumeName, | ||
| MountPath: mountPath, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| Volumes: []corev1.Volume{ | ||
| { | ||
| Name: volumeName, | ||
| VolumeSource: corev1.VolumeSource{ | ||
| ConfigMap: &corev1.ConfigMapVolumeSource{ | ||
| LocalObjectReference: corev1.LocalObjectReference{ | ||
| Name: experimentsDetails.ConfigMapName, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| _, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Create(context.Background(), helperPod, v1.CreateOptions{}) | ||
| if err != nil { | ||
| return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Reason: fmt.Sprintf("unable to create helper pod: %s", err.Error())} | ||
| } | ||
| return nil | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| ## Experiment Metadata | ||
|
|
||
| <table> | ||
| <tr> | ||
| <th> Name </th> | ||
| <th> Description </th> | ||
| <th> Documentation Link </th> | ||
| </tr> | ||
| <tr> | ||
| <td> Locust Load Generator </td> | ||
| <td> Locust is an open-source load testing tool that makes performance testing easy and productive for engineering teams. You can easily run load testing through a single Python script. Learn how to use k6 <a href="https://docs.locust.io/en/stable/">here</a> </td> | ||
| <td> <a href="https://litmuschaos.github.io/litmus/experiments/categories/load/locust-loadgen/"> Here </a> </td> | ||
| </tr> | ||
| </table> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| package experiment | ||
|
|
||
| import ( | ||
| "context" | ||
| "os" | ||
|
|
||
| "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" | ||
| litmusLIB "github.com/litmuschaos/litmus-go/chaoslib/litmus/locust-loadgen/lib" | ||
| "github.com/litmuschaos/litmus-go/pkg/clients" | ||
| "github.com/litmuschaos/litmus-go/pkg/events" | ||
| experimentEnv "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/environment" | ||
| experimentTypes "github.com/litmuschaos/litmus-go/pkg/load/locust-loadgen/types" | ||
| "github.com/litmuschaos/litmus-go/pkg/log" | ||
| "github.com/litmuschaos/litmus-go/pkg/probe" | ||
| "github.com/litmuschaos/litmus-go/pkg/result" | ||
| "github.com/litmuschaos/litmus-go/pkg/status" | ||
| "github.com/litmuschaos/litmus-go/pkg/types" | ||
| "github.com/litmuschaos/litmus-go/pkg/utils/common" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| // Experiment contains steps to inject chaos | ||
| func Experiment(ctx context.Context, clients clients.ClientSets) { | ||
|
|
||
| experimentsDetails := experimentTypes.ExperimentDetails{} | ||
| resultDetails := types.ResultDetails{} | ||
| eventsDetails := types.EventDetails{} | ||
| chaosDetails := types.ChaosDetails{} | ||
|
|
||
| //Fetching all the ENV passed from the runner pod | ||
| log.Infof("[PreReq]: Getting the ENV for the %v experiment", os.Getenv("EXPERIMENT_NAME")) | ||
| experimentEnv.GetENV(&experimentsDetails) | ||
|
|
||
| // Initialize the chaos attributes | ||
| types.InitialiseChaosVariables(&chaosDetails) | ||
|
|
||
| // Initialize Chaos Result Parameters | ||
| types.SetResultAttributes(&resultDetails, chaosDetails) | ||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| // Get values from chaosengine. Bail out upon error, as we haven't entered exp business logic yet | ||
| if err := types.GetValuesFromChaosEngine(&chaosDetails, clients, &resultDetails); err != nil { | ||
| log.Errorf("Unable to initialize the probes, err: %v", err) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| //Updating the chaos result in the beginning of experiment | ||
| log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) | ||
| if err := result.ChaosResult(&chaosDetails, clients, &resultDetails, "SOT"); err != nil { | ||
| log.Errorf("Unable to Create the Chaos Result, err: %v", err) | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
|
|
||
| // Set the chaos result uid | ||
| result.SetResultUID(&resultDetails, clients, &chaosDetails) | ||
|
|
||
| // generating the event in chaosresult to marked the verdict as awaited | ||
| msg := "experiment: " + experimentsDetails.ExperimentName + ", Result: Awaited" | ||
| types.SetResultEventAttributes(&eventsDetails, types.AwaitedVerdict, msg, "Normal", &resultDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResult") | ||
|
|
||
| //DISPLAY THE APP INFORMATION | ||
| log.InfoWithValues("[Info]: The application information is as follows", logrus.Fields{ | ||
| "Chaos Duration": experimentsDetails.ChaosDuration, | ||
| }) | ||
|
|
||
| // Calling AbortWatcher go routine, it will continuously watch for the abort signal and generate the required events and result | ||
| go common.AbortWatcher(experimentsDetails.ExperimentName, clients, &resultDetails, &chaosDetails, &eventsDetails) | ||
|
|
||
| //PRE-CHAOS APPLICATION STATUS CHECK | ||
| if chaosDetails.DefaultHealthCheck { | ||
| log.Info("[Status]: Verify that the AUT (Application Under Test) is running (pre-chaos)") | ||
| if err := status.AUTStatusCheck(clients, &chaosDetails); err != nil { | ||
| log.Errorf("Application status check failed, err: %v", err) | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, "AUT: Not Running", "Warning", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| // marking AUT as running, as we already checked the status of application under test | ||
| msg := "AUT: Running" | ||
|
|
||
| // run the probes in the pre-chaos check | ||
| if len(resultDetails.ProbeDetails) != 0 { | ||
|
|
||
| if err := probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PreChaos", &eventsDetails); err != nil { | ||
| log.Errorf("Probe Failed, err: %v", err) | ||
| msg := "AUT: Running, Probes: Unsuccessful" | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Warning", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
| msg = "AUT: Running, Probes: Successful" | ||
| } | ||
| // generating the events for the pre-chaos check | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, msg, "Normal", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| } | ||
|
|
||
| chaosDetails.Phase = types.ChaosInjectPhase | ||
| if err := litmusLIB.PrepareChaos(ctx, &experimentsDetails, clients, &resultDetails, &eventsDetails, &chaosDetails); err != nil { | ||
| log.Errorf("Chaos injection failed, err: %v", err) | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
|
|
||
| log.Infof("[Confirmation]: %v chaos has been injected successfully", experimentsDetails.ExperimentName) | ||
| resultDetails.Verdict = v1alpha1.ResultVerdictPassed | ||
| chaosDetails.Phase = types.PostChaosPhase | ||
|
|
||
| //POST-CHAOS APPLICATION STATUS CHECK | ||
| if chaosDetails.DefaultHealthCheck { | ||
| log.Info("[Status]: Verify that the AUT (Application Under Test) is running (post-chaos)") | ||
| if err := status.AUTStatusCheck(clients, &chaosDetails); err != nil { | ||
| log.Errorf("Application status check failed, err: %v", err) | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, "AUT: Not Running", "Warning", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
| } | ||
|
|
||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| // marking AUT as running, as we already checked the status of application under test | ||
| msg := "AUT: Running" | ||
|
|
||
| // run the probes in the post-chaos check | ||
| if len(resultDetails.ProbeDetails) != 0 { | ||
| if err := probe.RunProbes(ctx, &chaosDetails, clients, &resultDetails, "PostChaos", &eventsDetails); err != nil { | ||
| log.Errorf("Probes Failed, err: %v", err) | ||
| msg := "AUT: Running, Probes: Unsuccessful" | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Warning", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
| msg = "AUT: Running, Probes: Successful" | ||
| } | ||
|
|
||
| // generating post chaos event | ||
| types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, msg, "Normal", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| } | ||
|
|
||
| //Updating the chaosResult in the end of experiment | ||
| log.Infof("[The End]: Updating the chaos result of %v experiment (EOT)", experimentsDetails.ExperimentName) | ||
| if err := result.ChaosResult(&chaosDetails, clients, &resultDetails, "EOT"); err != nil { | ||
| log.Errorf("Unable to Update the Chaos Result, err: %v", err) | ||
| result.RecordAfterFailure(&chaosDetails, &resultDetails, err, clients, &eventsDetails) | ||
| return | ||
| } | ||
|
|
||
| // generating the event in chaosresult to mark the verdict as pass/fail | ||
| msg = "experiment: " + experimentsDetails.ExperimentName + ", Result: " + string(resultDetails.Verdict) | ||
| reason, eventType := types.GetChaosResultVerdictEvent(resultDetails.Verdict) | ||
| types.SetResultEventAttributes(&eventsDetails, reason, msg, eventType, &resultDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosResult") | ||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| msg := experimentsDetails.ExperimentName + " experiment has been " + string(resultDetails.Verdict) + "ed" | ||
| types.SetEngineEventAttributes(&eventsDetails, types.Summary, msg, "Normal", &chaosDetails) | ||
| events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine") | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
we can remove the application status check (pre-chaos) as it is not applicable for this experiment?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yes i will remove it