Merge pull request #25 from fasterci/ap/early_it_termination

more conditions to terminate integration test earlier

Author: apesternikov

testing/it_sidecar/it_sidecar.go

@@ -91,6 +91,7 @@ func contains(v []string, item string) bool {
 	}
 	return false
 }
+
 // listReadyApps converts a list returned from podsInformer.GetStore().List() to a map containing apps with ready status
 // app is determined by app label
 func listReadyApps(list []interface{}) (readypods, notReady []string) {
@@ -138,7 +139,7 @@ func listenForEvents(ctx context.Context, clientset *kubernetes.Clientset, onFai
 			return
 		}
 		log.Printf("EVENT %s %s %s %s\n", event.Namespace, event.InvolvedObject.Name, event.Reason, event.Message)
-		if event.Reason == "Failed" {
+		if event.Reason == "Failed" || event.Reason == "BackOff" {
 			onFailure(event)
 		}
 	}
@@ -370,7 +371,13 @@ func main() {
 	clientset = kubernetes.NewForConfigOrDie(config)
 	defer cleanup(clientset)
 
-	go stern.Run(ctx, *namespace, clientset)
+	go func() {
+		err := stern.Run(ctx, *namespace, clientset, allowErrors)
+		if err != nil {
+			log.Print(err)
+		}
+		cancel()
+	}()
 
 	listenForEvents(ctx, clientset, func(event *v1.Event) {
 		if !allowErrors {

testing/it_sidecar/stern/main.go

@@ -23,11 +23,11 @@ import (
 )
 
 // Run starts the main run loop
-func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset) error {
+func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset, allowErrors bool) error {
 
 	tails := make(map[string]*Tail)
 
-	err := Watch(ctx, clientset.CoreV1().Pods(namespace), RUNNING, labels.Everything(), func(p *Target) {
+	err := Watch(ctx, clientset.CoreV1().Pods(namespace), RUNNING, labels.Everything(), allowErrors, func(p *Target) {
 		id := p.GetID()
 		if tails[id] != nil {
 			return
@@ -45,7 +45,7 @@ func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset)
 		delete(tails, id)
 	})
 	if err != nil {
-		return fmt.Errorf("failed to set up watch: %v", err)
+		return fmt.Errorf("watch error: %v", err)
 	}
 
 	return nil

testing/it_sidecar/stern/watch.go

@@ -41,7 +41,7 @@ func (t *Target) GetID() string {
 // Watch starts listening to Kubernetes events and emits modified
 // containers/pods. The first result is targets added, the second is targets
 // removed
-func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState, labelSelector labels.Selector, onAdded, onRemoved func(*Target)) error {
+func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState, labelSelector labels.Selector, allowErrors bool, onAdded, onRemoved func(*Target)) error {
 	watcher, err := i.Watch(ctx, metav1.ListOptions{Watch: true, LabelSelector: labelSelector.String()})
 	if err != nil {
 		return fmt.Errorf("failed to set up watch: %s", err)
@@ -83,6 +83,12 @@ func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState
 
 					log.Print("container ", c.Name, " has state ", c.State)
 
+					if !allowErrors {
+						if t := c.State.Terminated; t != nil && t.ExitCode != 0 && t.Reason == "Error" {
+							return fmt.Errorf("container %s failed with exit code %d and reason '%s'", c.Name, t.ExitCode, t.Reason)
+						}
+					}
+
 					if containerState.Match(c.State) {
 						onAdded(&Target{
 							Namespace: pod.Namespace,