[fix] Set a maximum retries for Docker driver to avoid deadlock.

grafana · Nov 19, 2024 · 8cbc33c · 8cbc33c
1 parent ad3bde4
commit 8cbc33c
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 17 deletions.
diff --git a/clients/cmd/docker-driver/config.go b/clients/cmd/docker-driver/config.go
@@ -66,18 +66,16 @@ const (
 	defaultHostLabelName  = model.LabelName("host")
 )
 
-var (
-	defaultClientConfig = client.Config{
-		BatchWait: client.BatchWait,
-		BatchSize: client.BatchSize,
-		BackoffConfig: backoff.Config{
-			MinBackoff: client.MinBackoff,
-			MaxBackoff: client.MaxBackoff,
-			MaxRetries: client.MaxRetries,
-		},
-		Timeout: client.Timeout,
-	}
-)
+var defaultClientConfig = client.Config{
+	BatchWait: client.BatchWait,
+	BatchSize: client.BatchSize,
+	BackoffConfig: backoff.Config{
+		MinBackoff: client.MinBackoff,
+		MaxBackoff: client.MaxBackoff,
+		MaxRetries: client.MaxRetries,
+	},
+	Timeout: client.Timeout,
+}
 
 type config struct {
 	labels       model.LabelSet
@@ -153,7 +151,7 @@ func parseConfig(logCtx logger.Info) (*config, error) {
 	clientConfig.URL = flagext.URLValue{URL: url}
 
 	// parse timeout
-	if err := parseDuration(cfgTimeoutKey, logCtx, func(d time.Duration) { clientConfig.Timeout = d }); err != nil {
+	if err := parseDurationWithDefault(cfgTimeoutKey, logCtx, func(d time.Duration) { clientConfig.Timeout = d }, 10*time.Second); err != nil {
 		return nil, err
 	}
 
@@ -172,7 +170,7 @@ func parseConfig(logCtx logger.Info) (*config, error) {
 	if err := parseDuration(cfgMaxBackoffKey, logCtx, func(d time.Duration) { clientConfig.BackoffConfig.MaxBackoff = d }); err != nil {
 		return nil, err
 	}
-	if err := parseInt(cfgMaxRetriesKey, logCtx, func(i int) { clientConfig.BackoffConfig.MaxRetries = i }); err != nil {
+	if err := parseIntWithDefault(cfgMaxRetriesKey, logCtx, func(i int) { clientConfig.BackoffConfig.MaxRetries = i }, 2); err != nil {
 		return nil, err
 	}
 
@@ -350,6 +348,19 @@ func parseDuration(key string, logCtx logger.Info, set func(d time.Duration)) er
 	return nil
 }
 
+func parseDurationWithDefault(key string, logCtx logger.Info, set func(d time.Duration), defaultValue time.Duration) error {
+	if raw, ok := logCtx.Config[key]; ok {
+		val, err := time.ParseDuration(raw)
+		if err != nil {
+			return fmt.Errorf("%s: invalid option %s format: %s", driverName, key, raw)
+		}
+		set(val)
+	} else {
+		set(defaultValue)
+	}
+	return nil
+}
+
 func parseInt(key string, logCtx logger.Info, set func(i int)) error {
 	if raw, ok := logCtx.Config[key]; ok {
 		val, err := strconv.Atoi(raw)
@@ -361,6 +372,19 @@ func parseInt(key string, logCtx logger.Info, set func(i int)) error {
 	return nil
 }
 
+func parseIntWithDefault(key string, logCtx logger.Info, set func(i int), defaultValue int) error {
+	if raw, ok := logCtx.Config[key]; ok {
+		val, err := strconv.Atoi(raw)
+		if err != nil {
+			return fmt.Errorf("%s: invalid option %s format: %s", driverName, key, raw)
+		}
+		set(val)
+	} else {
+		set(defaultValue)
+	}
+	return nil
+}
+
 func relabelConfig(config string, lbs model.LabelSet) (model.LabelSet, error) {
 	relabelConfig := make([]*relabel.Config, 0)
 	if err := yaml.UnmarshalStrict([]byte(config), &relabelConfig); err != nil {

diff --git a/clients/cmd/docker-driver/driver.go b/clients/cmd/docker-driver/driver.go
@@ -81,14 +81,14 @@ func (d *driver) StartLogging(file string, logCtx logger.Info) error {
 		return err
 	}
 
-	keepFile, err := parseBoolean(cfgKeepFile, logCtx, false)
+	keepFile, err := parseBoolean(cfgKeepFile, logCtx, true)
 	if err != nil {
 		return err
 	}
 
 	var jsonl logger.Logger
 	if !noFile {
-		if err := os.MkdirAll(folder, 0755); err != nil {
+		if err := os.MkdirAll(folder, 0o755); err != nil {
 			return errors.Wrap(err, "error setting up logger dir")
 		}
 
@@ -102,7 +102,7 @@ func (d *driver) StartLogging(file string, logCtx logger.Info) error {
 	if err != nil {
 		return errors.Wrap(err, "error creating loki logger")
 	}
-	f, err := fifo.OpenFifo(context.Background(), file, syscall.O_RDONLY, 0700)
+	f, err := fifo.OpenFifo(context.Background(), file, syscall.O_RDONLY, 0o700)
 	if err != nil {
 		return errors.Wrapf(err, "error opening logger fifo: %q", file)
 	}