Fix e2e (#682)

* speedup e2e tests

* try to detect health of running app in e2e tests

Author: msaf1980

cmd/mockbackend/e2etesting.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
@@ -11,8 +12,10 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	merry2 "github.com/ansel1/merry"
@@ -281,21 +284,27 @@ func doTest(logger *zap.Logger, t *Query) []error {
 	return failures
 }
 
-func e2eTest(logger *zap.Logger, noapp bool) bool {
+func e2eTest(logger *zap.Logger, noapp, breakOnError bool) bool {
 	failed := false
 	logger.Info("will run test",
 		zap.Any("config", cfg.Test),
 	)
 	runningApps := make(map[string]*runner)
 	if !noapp {
+		wgStart := sync.WaitGroup{}
 		for i, c := range cfg.Test.Apps {
 			r := new(&cfg.Test.Apps[i], logger)
+			wgStart.Add(1)
 			runningApps[c.Name] = r
-			go r.Run()
+			go func() {
+				wgStart.Done()
+				r.Run()
+			}()
 		}
 
-		logger.Info("will sleep for 5 seconds to start all required apps")
-		time.Sleep(5 * time.Second)
+		wgStart.Wait()
+		logger.Info("will sleep for 1 seconds to start all required apps")
+		time.Sleep(1 * time.Second)
 	}
 
 	for _, t := range cfg.Test.Queries {
@@ -306,6 +315,22 @@ func e2eTest(logger *zap.Logger, noapp bool) bool {
 			logger.Error("test failed",
 				zap.Errors("failures", failures),
 			)
+			for _, v := range runningApps {
+				if !v.IsRunning() {
+					logger.Error("unexpected app crash", zap.Any("app", v))
+				}
+			}
+			if breakOnError {
+				for {
+					fmt.Print("Some queries was failed, press y for continue after debug test:")
+					in := bufio.NewScanner(os.Stdin)
+					in.Scan()
+					s := in.Text()
+					if s == "y" || s == "Y" {
+						break
+					}
+				}
+			}
 		} else {
 			logger.Info("test OK")
 		}
@@ -318,6 +343,9 @@ func e2eTest(logger *zap.Logger, noapp bool) bool {
 
 	if failed {
 		logger.Error("tests failed")
+		for _, v := range runningApps {
+			logger.Info("app out", zap.Any("app", v), zap.String("out", v.Out()))
+		}
 	} else {
 		logger.Info("All tests OK")
 	}

cmd/mockbackend/http_common.go

@@ -97,9 +97,7 @@ func copyResponse(src Response) Response {
 			Step:       src.Data[i].Step,
 		}
 
-		for j := range src.Data[i].Values {
-			dst.Data[i].Values[j] = src.Data[i].Values[j]
-		}
+		copy(dst.Data[i].Values, src.Data[i].Values)
 	}
 
 	return dst

cmd/mockbackend/main.go

@@ -40,6 +40,7 @@ func main() {
 	testonly := flag.Bool("testonly", false, "run only unit test")
 	noapp := flag.Bool("noapp", false, "do not run application")
 	test := flag.Bool("test", false, "run unit test if present")
+	breakOnError := flag.Bool("break", false, "break and wait user response if request failed")
 	flag.Parse()
 	logger, err := zap.NewProduction()
 	if err != nil {
@@ -67,6 +68,7 @@ func main() {
 
 	httpServers := make([]*http.Server, 0)
 	wg := sync.WaitGroup{}
+	wgStart := sync.WaitGroup{}
 	if !*testonly {
 		for _, c := range cfg.Listeners {
 			logger := logger.With(zap.String("listener", c.Address))
@@ -95,28 +97,31 @@ func main() {
 			mux.HandleFunc("/metrics/find/", listener.findHandler)
 
 			wg.Add(1)
+			wgStart.Add(1)
 			server := &http.Server{
 				Addr:    listener.Address,
 				Handler: mux,
 			}
 			go func(h *http.Server) {
+				wgStart.Done()
 				err = h.ListenAndServe()
-				if err != nil {
+				if err != nil && err != http.ErrServerClosed {
 					logger.Error("failed to start server",
 						zap.Error(err),
 					)
 				}
 				wg.Done()
 			}(server)
 
+			wgStart.Wait()
 			httpServers = append(httpServers, server)
 		}
 		logger.Info("all listeners started")
 	}
 
 	failed := false
 	if cfg.Test != nil && (*test || *testonly) {
-		failed = e2eTest(logger, *noapp)
+		failed = e2eTest(logger, *noapp, *breakOnError)
 	}
 
 	if !*testonly {

cmd/mockbackend/runner.go

@@ -4,17 +4,22 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"sync"
 
+	"github.com/tevino/abool"
 	"go.uber.org/zap"
 )
 
 type runner struct {
 	App
 
-	isRunning bool
+	out string
+
+	isRunning *abool.AtomicBool
 	ctx       context.Context
 	cancel    context.CancelFunc
 	logger    *zap.Logger
+	wg        sync.WaitGroup
 }
 
 func new(config *App, logger *zap.Logger) *runner {
@@ -23,13 +28,17 @@ func new(config *App, logger *zap.Logger) *runner {
 		logger: logger.With(
 			zap.String("name", config.Name),
 		),
+		isRunning: abool.New(),
 	}
 
 	return r
 }
 
 func (r *runner) Run() {
-	if r.isRunning {
+	r.wg.Add(1)
+	defer r.wg.Done()
+
+	if !r.isRunning.SetToIf(false, true) {
 		r.logger.Fatal("app is already running")
 	}
 	r.logger.Debug("will start application",
@@ -39,20 +48,28 @@ func (r *runner) Run() {
 
 	cmd := exec.CommandContext(r.ctx, r.Binary, r.Args...)
 	out, err := cmd.CombinedOutput()
+	r.out = string(out)
 	if err != nil {
 		r.logger.Error("error running program",
 			zap.Any("config", r.App),
 			zap.String("output", "will follow next"),
 			zap.Error(err),
 		)
-		fmt.Print(string(out))
+		fmt.Print(r.out)
 	}
 
-	r.isRunning = true
+	r.isRunning.UnSet()
 }
 
 func (r *runner) Finish() {
 	r.cancel()
+	r.wg.Wait()
+}
+
+func (r *runner) IsRunning() bool {
+	return r.isRunning.IsSet()
+}
 
-	r.isRunning = false
+func (r *runner) Out() string {
+	return r.out
 }

e2e_test.sh

@@ -2,6 +2,12 @@
 
 TESTS=$(ls cmd/mockbackend/testcases/)
 
+OPTIONS=""
+if [[ "$1" == "-break" ]]; then
+	OPTIONS="${OPTIONS} -break"
+	shift
+fi
+
 if [[ ! -z "${1}" ]]; then
 	TESTS="${1}"
 fi
@@ -14,15 +20,15 @@ for t in ${TESTS}; do
 	fi
 
 	echo "RUNNING TEST ${t}"
-	./mockbackend -test -config cmd/mockbackend/testcases/${t}/${t}.yaml
+	./mockbackend ${OPTIONS} -test -config cmd/mockbackend/testcases/${t}/${t}.yaml
 	status=$?
 
 	if [[ "${TRAVIS}" == "true" ]]; then
 		travis_time_finish
 		travis_fold end "test_${t}"
 	fi
 
-	sleep 10
+	sleep 2
 	if [[ ${status} -ne 0 ]]; then
 		FAILED_TESTS="${FAILED_TESTS} ${t}"
 		echo "test_${t}: FAIL"

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.7.1
 	github.com/subosito/gotenv v1.4.0 // indirect
+	github.com/tevino/abool v1.2.0 // indirect
 	github.com/tinylib/msgp v1.1.6
 	github.com/valyala/fastjson v1.6.3
 	github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f

go.sum

@@ -494,6 +494,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
 github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs=
 github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo=
+github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
+github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
 github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
 github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=