reduce duplicate reporting of errors (#45)

pkg/f1/run/test_runner.go

@@ -89,13 +89,13 @@ func (r *Run) Do() *RunResult {
 	r.configureLogging()
 
 	metrics.Instance().Reset()
-	var err error
-	r.activeScenario, err = testing.NewActiveScenarios(r.Options.Scenario, r.Options.Env, testing.GetScenario(r.Options.Scenario), 0)
+	var setupSuccessful bool
+	r.activeScenario, setupSuccessful = testing.NewActiveScenarios(r.Options.Scenario, r.Options.Env, testing.GetScenario(r.Options.Scenario), 0)
 	r.pushMetrics()
 	fmt.Println(r.result.Setup())
 
-	if err != nil {
-		return r.fail(err, "setup failed")
+	if !setupSuccessful {
+		return r.fail("setup failed")
 	}
 
 	// set initial started timestamp so that the progress trackers work
@@ -135,9 +135,9 @@ func (r *Run) teardown() {
 	}
 
 	if r.activeScenario.TeardownFn != nil {
-		err := r.activeScenario.Run(metrics.TeardownResult, "teardown", "0", "teardown", r.activeScenario.TeardownFn)
-		if err != nil {
-			r.fail(err, "teardown failed")
+		successful := r.activeScenario.Run(metrics.TeardownResult, "teardown", "0", "teardown", r.activeScenario.TeardownFn)
+		if !successful {
+			r.fail("teardown failed")
 		}
 	} else {
 		log.Infof("nil teardown function for scenario %s", r.Options.Scenario)
@@ -308,9 +308,8 @@ func (r *Run) runWorker(input <-chan int32, stop <-chan struct{}, wg *sync.WaitG
 			trace.Event("Received work (%v) from Channel 'doWork' iteration (%v)", worker, iteration)
 			atomic.AddInt32(&r.busyWorkers, 1)
 			for _, stage := range r.activeScenario.Stages {
-				err := r.activeScenario.Run(metrics.IterationResult, stage.Name, worker, fmt.Sprint(iteration), stage.RunFn)
-				if err != nil {
-					log.WithError(err).Error("failed iteration run")
+				successful := r.activeScenario.Run(metrics.IterationResult, stage.Name, worker, fmt.Sprint(iteration), stage.RunFn)
+				if !successful {
 					atomic.AddInt32(&r.failures, 1)
 				}
 			}
@@ -330,8 +329,8 @@ func (r *Run) runWorker(input <-chan int32, stop <-chan struct{}, wg *sync.WaitG
 	}
 }
 
-func (r *Run) fail(err error, message string) *RunResult {
-	r.result.AddError(errors.Wrap(err, message))
+func (r *Run) fail(message string) *RunResult {
+	r.result.AddError(fmt.Errorf(message))
 	return &r.result
 }
 

pkg/f1/testing/active_scenario.go

@@ -1,7 +1,6 @@
 package testing
 
 import (
-	"errors"
 	"runtime/debug"
 	"sync"
 	"time"
@@ -23,7 +22,7 @@ type ActiveScenario struct {
 	m                   *metrics.Metrics
 }
 
-func NewActiveScenarios(name string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, error) {
+func NewActiveScenarios(name string, env map[string]string, fn MultiStageSetupFn, autoTeardownIdleDuration time.Duration) (*ActiveScenario, bool) {
 
 	s := &ActiveScenario{
 		Name:              name,
@@ -32,7 +31,7 @@ func NewActiveScenarios(name string, env map[string]string, fn MultiStageSetupFn
 		AutoTeardownAfter: autoTeardownIdleDuration,
 		m:                 metrics.Instance(),
 	}
-	err := s.Run(metrics.SetupResult, "setup", "0", "setup", func(t *T) {
+	successful := s.Run(metrics.SetupResult, "setup", "0", "setup", func(t *T) {
 		s.Stages, s.TeardownFn = fn(t)
 
 		if autoTeardownIdleDuration > 0 {
@@ -50,7 +49,7 @@ func NewActiveScenarios(name string, env map[string]string, fn MultiStageSetupFn
 		log.Infof("Added active scenario %s",
 			s.id)
 	})
-	return s, err
+	return s, successful
 }
 
 func (s *ActiveScenario) AutoTeardown() *CancellableTimer {
@@ -67,7 +66,8 @@ func (s *ActiveScenario) SetAutoTeardown(timer *CancellableTimer) {
 	s.autoTeardownTimer = timer
 }
 
-func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, f func(t *T)) error {
+// Run performs a single iteration of the test. It returns `true` if the test was successful, `false` otherwise.
+func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string, f func(t *T)) bool {
 	t := NewT(s.env, vu, iter, s.Name)
 	start := time.Now()
 	done := make(chan struct{})
@@ -81,22 +81,18 @@ func (s *ActiveScenario) Run(metric metrics.MetricType, stage, vu, iter string,
 	// wait for completion
 	<-done
 	s.m.Record(metric, s.Name, stage, metrics.Result(t.HasFailed()), time.Since(start).Nanoseconds())
-	if t.HasFailed() {
-		return errors.New("failed")
-	}
-	return nil
+	return !t.HasFailed()
 }
 
 func (s *ActiveScenario) checkResults(t *T, done chan<- struct{}) {
 	r := recover()
 	if r != nil {
-		t.Fail()
 		err, isError := r.(error)
 		if isError {
-			t.Log.WithError(err).Errorf("panic in `%s` test scenario on iteration `%s` for user `%s`", t.Scenario, t.Iteration, t.VirtualUser)
+			t.FailWithError(err)
 			debug.PrintStack()
 		} else {
-			t.Log.Errorf("panic in `%s` test scenario on iteration `%s` for user `%s`: %v", t.Scenario, t.Iteration, t.VirtualUser, r)
+			t.Errorf("panic in test iteration: %v", err)
 		}
 	}
 	close(done)
@@ -105,9 +101,9 @@ func (s *ActiveScenario) checkResults(t *T, done chan<- struct{}) {
 func (s *ActiveScenario) autoTeardown() {
 	log.Warn("Teardown not called - triggering timed teardown")
 	if s.TeardownFn != nil {
-		err := s.Run(metrics.TeardownResult, "teardown", "0", "teardown", s.TeardownFn)
-		if err != nil {
-			log.WithError(err).Error("auto teardown failed")
+		successful := s.Run(metrics.TeardownResult, "teardown", "0", "teardown", s.TeardownFn)
+		if !successful {
+			log.Error("auto teardown failed")
 		}
 	}
 	activeScenarios.Delete(s.id)

pkg/f1/testing/t.go

@@ -38,17 +38,24 @@ func NewT(env map[string]string, vu, iter string, scenarioName string) *T {
 }
 
 func (t *T) Errorf(format string, args ...interface{}) {
-	t.Fail()
+	atomic.StoreInt64(&t.failed, int64(1))
 	t.Log.Errorf(format, args...)
 }
 
 func (t *T) FailNow() {
-	t.Fail()
+	atomic.StoreInt64(&t.failed, int64(1))
+	t.Log.Errorf("test failed and stopped")
 	runtime.Goexit()
 }
 
 func (t *T) Fail() {
 	atomic.StoreInt64(&t.failed, int64(1))
+	t.Log.Errorf("test failed")
+}
+
+func (t *T) FailWithError(err error) {
+	atomic.StoreInt64(&t.failed, int64(1))
+	t.Log.WithError(err).Errorf("test failed due to %s", err.Error())
 }
 
 func (t *T) HasFailed() bool {