fix: docker reconnect policy

FMotalleb · Jul 13, 2024 · 1b6b5e1 · 1b6b5e1
1 parent e405429
commit 1b6b5e1
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/config/job_valdiator.go b/config/job_valdiator.go
@@ -207,7 +207,7 @@ func dockerValidation(s *JobEvent, log *logrus.Entry) error {
 		log.Debug("error limit will be set to 1")
 	}
 	if s.Docker.ErrorLimitPolicy == "" {
-		log.Info("error limit set to non-zero number but no error policy was specified, using default policy (reconnect)")
+		log.Info("no error policy was specified, using default policy (reconnect)")
 	}
 	if !utils.NewList("", event.GiveUp, event.Kill, event.Reconnect).Contains(s.Docker.ErrorLimitPolicy) {
 		err := fmt.Errorf("given error limit policy: %#v is not allowed, possible error policies are (give-up,kill,reconnect)", s.Docker.ErrorLimitPolicy)

diff --git a/core/event/docker.go b/core/event/docker.go
@@ -48,7 +48,7 @@ func NewDockerEvent(
 		connection:       connection,
 		containerMatcher: *regexp.MustCompile(containerMatcher),
 		imageMatcher:     *regexp.MustCompile(imageMatcher),
-		actions:          toAction(logger, actions),
+		actions:          toAction(actions),
 		labels:           reshapeLabelMatcher(labels),
 		errorThreshold:   errorLimit,
 		errorPolicy:      errorPolicy,
@@ -91,18 +91,22 @@ func (de *DockerEvent) BuildTickChannel() <-chan any {
 					case Kill:
 						de.log.Fatalf("Received more than %d consecutive errors from docker, marking instance as unstable and killing in return, this may happen due to dockerd restarting", errs)
 					case Reconnect:
-						de.log.Fatalf("Received more than %d consecutive errors from docker, marking instance as unstable and retry connecting to docker", errs)
+						de.log.Warnf("Received more than %d consecutive errors from docker, marking instance as unstable and retry connecting to docker", errs)
+						for range de.BuildTickChannel() {
+							notifyChan <- nil
+						}
 					default:
 						de.log.Fatalf("unexpected event.ErrorLimitPolicy: %#v, valid options are (kill,giv-up,reconnect)", de.errorPolicy)
 					}
+					errCount.Set(0)
 				}
 				if de.errorThrottle > 0 {
 					time.Sleep(de.errorThrottle)
 				}
 			case event := <-msg:
 				de.log.Trace("received an event from docker: ", event)
 				if de.matches(&event) {
-					notifyChan <- false
+					notifyChan <- nil
 				}
 				errCount.Set(0)
 			}
@@ -140,7 +144,7 @@ func reshapeLabelMatcher(labels map[string]string) map[string]regexp.Regexp {
 	return res
 }
 
-func toAction(log *logrus.Entry, acts []string) *utils.List[events.Action] {
+func toAction(acts []string) *utils.List[events.Action] {
 	actions := utils.NewList[events.Action]()
 	for _, act := range acts {
 		actions.Add(events.Action(act))