Skip to content

Commit

Permalink
Do not set stdout error on EOF retry
Browse files Browse the repository at this point in the history
stdoutErr is use to determine `errDetail` that's used in `kw.UpdateBasicStatus(WorkStateFailed, errDetail, stdout.Size())`

in case where we retried 5 time and did not read any new log messages it is not an error it's the expected happy path so we should not set stdoutErr
  • Loading branch information
TheRealHaoLiu authored and AaronH88 committed Feb 7, 2024
1 parent 7eaf7f2 commit cae6862
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions pkg/workceptor/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,8 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout
kw.GetWorkceptor().nc.GetLogger().Info(
"Context was canceled while reading logs for pod %s/%s. Assuming pod has finished",
podNamespace,
podName)
podName,
)

return
}
Expand All @@ -263,9 +264,16 @@ func (kw *kubeUnit) kubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout

break
}
*stdoutErr = err

kw.GetWorkceptor().nc.GetLogger().Error("Error reading from pod %s/%s: %s", podNamespace, podName, err)

// At this point we exausted all retries, every retry we either failed to read OR we read but did not get newer msg
// If we got a EOF on the last retry we assume that we read everything and we can stop the loop
// we ASSUME this is the happy path.
if err != io.EOF {
*stdoutErr = err
}

return
}

Expand Down Expand Up @@ -490,6 +498,7 @@ func (kw *kubeUnit) runWorkUsingLogger() {

if podName == "" {
// create new pod if ked.PodName is empty
// TODO: add retry logic to make this more resilient to transient errors
if err := kw.createPod(nil); err != nil {
if err != ErrPodCompleted {
errMsg := fmt.Sprintf("Error creating pod: %s", err)
Expand Down

0 comments on commit cae6862

Please sign in to comment.