Add max failures option (#107) (#139)

* Add max failures option

* Add tests

---------

Co-authored-by: Adrien Sambres <[email protected]>
Co-authored-by: Dmytro Vovk <[email protected]>

config-file-example.yaml

@@ -22,6 +22,8 @@ limits:                   # Global limits to apply to the program execution
   max-duration: 5s        # Equivalent to --max-duration flag, will stop the execution once this duration is completed if the program is still running
   concurrency: 50         # Equivalent to --concurrency flag, will limit the concurrency of the running iterations
   max-iterations: 1000    # Equivalent to --max-iterations flag, will run no more then the number of iterations specified here
+  max-failures: 0         # Equivalent to --max-failures flag, the load test will fail if the number of failures is superior to the number specified here
+  max-failures-rate: 0    # Equivalent to --max-failures-rate flag, the load test will fail if the percentage of failures is superior to the percentage specified here
   ignore-dropped: true    # Equivalent to --ignore-dropped flag, drop requests will not fail the run
 schedule:
   stage-start: "2020-12-10T09:00:00+00:00"  # Restarting an execution will skip the stages which were completed, based on the stage duration and this field

internal/options/run_options.go

@@ -13,6 +13,8 @@ type RunOptions struct {
 	Verbose             bool
 	VerboseFail         bool
 	MaxIterations       int32
+	MaxFailures         int
+	MaxFailuresRate     int
 	RegisterLogHookFunc logging.RegisterLogHookFunc
 	IgnoreDropped       bool
 }

internal/run/result.go

@@ -18,6 +18,8 @@ type RunResult struct {
 	SuccessfulIterationDurations DurationPercentileMap
 	FailedIterationCount         uint64
 	FailedIterationDurations     DurationPercentileMap
+	MaxFailedIterations          int
+	MaxFailedIterationsRate      int
 	startTime                    time.Time
 	TestDuration                 time.Duration
 	LogFile                      string
@@ -173,7 +175,12 @@ func (r *RunResult) String() string {
 func (r *RunResult) Failed() bool {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
-	return r.Error() != nil || r.FailedIterationCount > 0 || (!r.IgnoreDropped && r.DroppedIterationCount > 0)
+
+	return r.Error() != nil ||
+		(!r.IgnoreDropped && r.DroppedIterationCount > 0) ||
+		(r.MaxFailedIterations == 0 && r.MaxFailedIterationsRate == 0 && r.FailedIterationCount > 0) ||
+		(r.MaxFailedIterations > 0 && r.FailedIterationCount > uint64(r.MaxFailedIterations)) ||
+		(r.MaxFailedIterationsRate > 0 && (r.FailedIterationCount*100/r.Iterations() > uint64(r.MaxFailedIterationsRate)))
 }
 
 func (r *RunResult) Progress() string {

internal/run/run_cmd.go

@@ -50,6 +50,8 @@ func Cmd(s *scenarios.Scenarios, builders []api.Builder, hookFunc logging.Regist
 			triggerCmd.Flags().DurationP("max-duration", "d", time.Second, "--max-duration 1s (stop after 1 second)")
 			triggerCmd.Flags().IntP("concurrency", "c", 100, "--concurrency 2 (allow at most 2 groups of iterations to run concurrently)")
 			triggerCmd.Flags().Int32P("max-iterations", "i", 0, "--max-iterations 100 (stop after 100 iterations, regardless of remaining duration)")
+			triggerCmd.Flags().Int("max-failures", 0, "--max-failures 10 (load test will fail if more than 10 errors occured, default is 0)")
+			triggerCmd.Flags().Int("max-failures-rate", 0, "--max-failures-rate 5 (load test will fail if more than 5\\% requests failed, default is 0)")
 
 			triggerCmd.Flags().AddFlagSet(t.Flags)
 			runCmd.AddCommand(triggerCmd)
@@ -72,12 +74,16 @@ func runCmdExecute(s *scenarios.Scenarios, t api.Builder, hookFunc logging.Regis
 		var duration time.Duration
 		var concurrency int
 		var maxIterations int32
+		var maxFailures int
+		var maxFailuresRate int
 		var ignoreDropped bool
 		if t.IgnoreCommonFlags {
 			scenarioName = trig.Options.Scenario
 			duration = trig.Options.MaxDuration
 			concurrency = trig.Options.Concurrency
 			maxIterations = trig.Options.MaxIterations
+			maxFailures = trig.Options.MaxFailures
+			maxFailures = trig.Options.MaxFailuresRate
 			ignoreDropped = trig.Options.IgnoreDropped
 		} else {
 			scenarioName = args[0]
@@ -93,6 +99,14 @@ func runCmdExecute(s *scenarios.Scenarios, t api.Builder, hookFunc logging.Regis
 			if err != nil {
 				return errors.New(fmt.Sprintf("Invalid maxIterations value: %s", err))
 			}
+			maxFailures, err = cmd.Flags().GetInt("max-failures")
+			if err != nil {
+				return errors.New(fmt.Sprintf("Invalid maxFailures value: %s", err))
+			}
+			maxFailuresRate, err = cmd.Flags().GetInt("max-failures-rate")
+			if err != nil {
+				return errors.New(fmt.Sprintf("Invalid maxFailuresRate value: %s", err))
+			}
 			ignoreDropped, err = cmd.Flags().GetBool("ignore-dropped")
 			if err != nil {
 				return errors.New(fmt.Sprintf("Invalid ignore-dropped value: %s", err))
@@ -116,6 +130,8 @@ func runCmdExecute(s *scenarios.Scenarios, t api.Builder, hookFunc logging.Regis
 			Verbose:             verbose,
 			VerboseFail:         verboseFail,
 			MaxIterations:       maxIterations,
+			MaxFailures:         maxFailures,
+			MaxFailuresRate:     maxFailuresRate,
 			RegisterLogHookFunc: hookFunc,
 			IgnoreDropped:       ignoreDropped,
 		}, trig)

internal/run/run_cmd_test.go

@@ -67,6 +67,8 @@ type TestParam struct {
 	expectedDroppedIterations uint64
 	expectedFailure           bool
 	maxIterations             int32
+	maxFailures               int
+	maxFailuresRate           int
 	stages                    string
 	iterationFrequency        string
 	distributionType          string
@@ -600,3 +602,75 @@ func TestFailureCounts(t *testing.T) {
 		the_iteration_metric_has_n_results(5, "success").and().
 		the_iteration_metric_has_n_results(5, "fail")
 }
+
+func TestParameterisedMaxFailures(t *testing.T) {
+	for _, test := range []TestParam{
+		{
+			name:            "pass with 5 max failures",
+			maxFailures:     5,
+			expectedFailure: false,
+		},
+		{
+			name:            "pass with superior max failures",
+			maxFailures:     6,
+			expectedFailure: false,
+		},
+		{
+			name:            "fails with inferior max failures",
+			maxFailures:     3,
+			expectedFailure: true,
+		},
+		{
+			name:            "pass with 50% max failures rate",
+			maxFailuresRate: 50,
+			expectedFailure: false,
+		},
+		{
+			name:            "pass with superior max failures rate",
+			maxFailuresRate: 60,
+			expectedFailure: false,
+		},
+		{
+			name:            "fails with inferior max failures rate",
+			maxFailuresRate: 30,
+			expectedFailure: true,
+		},
+		{
+			name:            "pass with inferior max failures and max failures rate",
+			maxFailures:     3,
+			maxFailuresRate: 30,
+			expectedFailure: true,
+		},
+		{
+			name:            "fails with inferior max failures rate and superior max failures",
+			maxFailures:     6,
+			maxFailuresRate: 30,
+			expectedFailure: true,
+		},
+		{
+			name:            "fails with inferior max failures and superior max failures rate",
+			maxFailures:     3,
+			maxFailuresRate: 60,
+			expectedFailure: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			given, when, then := NewRunTestStage(t)
+
+			given.
+				a_rate_of("10/100ms").and().
+				a_max_failures_of(test.maxFailures).and().
+				a_max_failures_rate_of(test.maxFailuresRate).and().
+				a_duration_of(100 * time.Millisecond).and().
+				a_test_scenario_that_fails_intermittently().and().
+				a_distribution_type("none")
+
+			when.i_execute_the_run_command()
+
+			then.
+				the_iteration_metric_has_n_results(5, "success").and().
+				the_iteration_metric_has_n_results(5, "fail").and().
+				the_command_finished_with_failure_of(test.expectedFailure)
+		})
+	}
+}

internal/run/run_stage_test.go

@@ -50,6 +50,8 @@ type RunTestStage struct {
 	assert                 *assert.Assertions
 	rate                   string
 	maxIterations          int32
+	maxFailures            int
+	maxFailuresRate        int
 	triggerType            TriggerType
 	stages                 string
 	frequency              string
@@ -98,6 +100,16 @@ func (s *RunTestStage) a_concurrency_of(concurrency int) *RunTestStage {
 	return s
 }
 
+func (s *RunTestStage) a_max_failures_of(maxFailures int) *RunTestStage {
+	s.maxFailures = maxFailures
+	return s
+}
+
+func (s *RunTestStage) a_max_failures_rate_of(maxFailuresRate int) *RunTestStage {
+	s.maxFailuresRate = maxFailuresRate
+	return s
+}
+
 func (s *RunTestStage) a_config_file_location_of(commandsFile string) *RunTestStage {
 	s.configFile = commandsFile
 	return s
@@ -125,6 +137,8 @@ func (s *RunTestStage) i_execute_the_run_command() *RunTestStage {
 			MaxDuration:         s.duration,
 			Concurrency:         s.concurrency,
 			MaxIterations:       s.maxIterations,
+			MaxFailures:         s.maxFailures,
+			MaxFailuresRate:     s.maxFailuresRate,
 			RegisterLogHookFunc: fluentd_hook.AddFluentdLoggingHook,
 		},
 		s.build_trigger())
@@ -172,6 +186,12 @@ func (s *RunTestStage) the_command_should_fail() *RunTestStage {
 	return s
 }
 
+func (s *RunTestStage) the_command_should_succeeded() *RunTestStage {
+	s.assert.NotNil(s.runResult)
+	s.assert.Equal(false, s.runResult.Failed())
+	return s
+}
+
 func (s *RunTestStage) a_test_scenario_that_always_fails() *RunTestStage {
 	s.scenario = uuid.New().String()
 	s.f1.Add(s.scenario, func(t *f1_testing.T) (fn f1_testing.RunFn) {

internal/run/test_runner.go

@@ -48,6 +48,8 @@ func NewRun(options options.RunOptions, t *api.Trigger) (*Run, error) {
 		run.Options.RegisterLogHookFunc = logging.NoneRegisterLogHookFunc
 	}
 	run.result.IgnoreDropped = options.IgnoreDropped
+	run.result.MaxFailedIterations = options.MaxFailures
+	run.result.MaxFailedIterationsRate = options.MaxFailuresRate
 
 	progressRunner, _ := raterun.New(func(rate time.Duration, t time.Time) {
 		run.gatherProgressMetrics(rate)

internal/testdata/config-file.yaml

@@ -6,6 +6,8 @@ limits:
   max-duration: 5s
   concurrency: 50
   max-iterations: 1000
+  max-failures: 0
+  max-failures-rate: 0
   ignore-dropped: true
 stages:
   - duration: 500ms

internal/trigger/api/api.go

@@ -37,13 +37,15 @@ type Trigger struct {
 }
 
 type Options struct {
-	MaxDuration   time.Duration
-	Concurrency   int
-	Verbose       bool
-	VerboseFail   bool
-	MaxIterations int32
-	IgnoreDropped bool
-	Scenario      string
+	MaxDuration     time.Duration
+	Concurrency     int
+	Verbose         bool
+	VerboseFail     bool
+	MaxIterations   int32
+	MaxFailures     int
+	MaxFailuresRate int
+	IgnoreDropped   bool
+	Scenario        string
 }
 
 type Rates struct {

internal/trigger/file/file_parser.go

@@ -24,10 +24,12 @@ type Schedule struct {
 }
 
 type Limits struct {
-	MaxDuration   *time.Duration `yaml:"max-duration"`
-	Concurrency   *int           `yaml:"concurrency"`
-	MaxIterations *int32         `yaml:"max-iterations"`
-	IgnoreDropped *bool          `yaml:"ignore-dropped"`
+	MaxDuration     *time.Duration `yaml:"max-duration"`
+	Concurrency     *int           `yaml:"concurrency"`
+	MaxIterations   *int32         `yaml:"max-iterations"`
+	MaxFailures     *int           `yaml:"max-failures"`
+	MaxFailuresRate *int           `yaml:"max-failures-rate"`
+	IgnoreDropped   *bool          `yaml:"ignore-dropped"`
 }
 
 type Stage struct {
@@ -85,6 +87,8 @@ func parseConfigFile(fileContent []byte, now time.Time) (*runnableStages, error)
 		maxDuration:         *validatedConfigFile.Limits.MaxDuration,
 		concurrency:         *validatedConfigFile.Limits.Concurrency,
 		maxIterations:       *validatedConfigFile.Limits.MaxIterations,
+		maxFailures:         *validatedConfigFile.Limits.MaxFailures,
+		maxFailuresRate:     *validatedConfigFile.Limits.MaxFailuresRate,
 		ignoreDropped:       *validatedConfigFile.Limits.IgnoreDropped,
 	}, nil
 }
@@ -194,6 +198,14 @@ func (c *ConfigFile) validateCommonFields() (*ConfigFile, error) {
 		return nil, fmt.Errorf("missing stages")
 	}
 
+	if c.Limits.MaxFailures == nil {
+		var maxFailures = 0
+		c.Limits.MaxFailures = &maxFailures
+	}
+	if c.Limits.MaxFailuresRate == nil {
+		var maxFailuresRate = 0
+		c.Limits.MaxFailuresRate = &maxFailuresRate
+	}
 	if c.Default.Concurrency == nil {
 		c.Default.Concurrency = c.Limits.Concurrency
 	}

internal/trigger/file/file_parser_test.go

@@ -347,6 +347,37 @@ stages:
 			expectedRates:             []int{2, 2, 2, 2, 2},
 			expectedParameters:        map[string]string{"FOO": "bar"},
 		},
+		{
+			testName: "Include max failures",
+			fileContent: `scenario: template
+limits:
+  max-duration: 1m
+  concurrency: 50
+  max-iterations: 100
+  max-failures: 10
+  max-failures-rate: 5
+  ignore-dropped: true
+stages:
+- duration: 5s
+  mode: constant
+  rate: 6/s
+  jitter: 0
+  distribution: none
+  parameters:
+    FOO: bar
+`,
+			expectedScenario:          "template",
+			expectedMaxDuration:       1 * time.Minute,
+			expectedConcurrency:       50,
+			expectedMaxIterations:     100,
+			expectedMaxFailures:       10,
+			expectedMaxFailuresRate:   5,
+			expectedIgnoreDropped:     true,
+			expectedTotalDuration:     5 * time.Second,
+			expectedIterationDuration: 1 * time.Second,
+			expectedRates:             []int{6, 6, 6, 6, 6, 6},
+			expectedParameters:        map[string]string{"FOO": "bar"},
+		},
 	} {
 		t.Run(test.testName, func(t *testing.T) {
 			now, _ := time.Parse(time.RFC3339, "2020-12-10T10:00:00+00:00")
@@ -631,6 +662,8 @@ type testData struct {
 	expectedMaxDuration       time.Duration
 	expectedIgnoreDropped     bool
 	expectedMaxIterations     int32
+	expectedMaxFailures       int
+	expectedMaxFailuresRate   int
 	expectedConcurrency       int
 	expectedUsersConcurrency  int
 	expectedRates             []int

internal/trigger/file/file_rate.go

@@ -19,6 +19,8 @@ type runnableStages struct {
 	maxDuration         time.Duration
 	concurrency         int
 	maxIterations       int32
+	maxFailures         int
+	maxFailuresRate     int
 	ignoreDropped       bool
 }
 
@@ -54,11 +56,13 @@ func FileRate() api.Builder {
 				Description: fmt.Sprintf("%d different stages", len(runnableStages.stages)),
 				Duration:    runnableStages.stagesTotalDuration,
 				Options: api.Options{
-					Scenario:      runnableStages.scenario,
-					MaxDuration:   runnableStages.maxDuration,
-					Concurrency:   runnableStages.concurrency,
-					MaxIterations: runnableStages.maxIterations,
-					IgnoreDropped: runnableStages.ignoreDropped,
+					Scenario:        runnableStages.scenario,
+					MaxDuration:     runnableStages.maxDuration,
+					Concurrency:     runnableStages.concurrency,
+					MaxIterations:   runnableStages.maxIterations,
+					MaxFailures:     runnableStages.maxFailures,
+					MaxFailuresRate: runnableStages.maxFailuresRate,
+					IgnoreDropped:   runnableStages.ignoreDropped,
 				},
 			}, nil
 		},