Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: retry querying logs on creation failure #71

Merged
merged 1 commit into from
Dec 19, 2023
Merged

fix: retry querying logs on creation failure #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions api/log/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"github.com/grafana/loki/pkg/logqlmodel"
"github.com/grafana/loki/pkg/util/unmarshal"
"github.com/prometheus/common/config"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
)

type Client struct {
Expand Down Expand Up @@ -89,6 +91,38 @@ func (c *Client) QueryRange(ctx context.Context, out output.LogOutput, q Query)
return printResult(resp.Data.Result, out)
}

// QueryRangeWithRetry queries logs within a specific time range with a retry
// in case of an error or not finding any logs.
func (c *Client) QueryRangeWithRetry(ctx context.Context, out output.LogOutput, q Query) error {
return retry.OnError(
wait.Backoff{
Steps: 5,
Duration: 200 * time.Millisecond,
Factor: 2.0,
Jitter: 0.1,
Cap: 10 * time.Second,
},
func(err error) bool {
// retry regardless of the error
return true
},
func() error {
resp, err := c.Client.QueryRange(q.QueryString, q.Limit, q.Start, q.End, q.Direction, q.Step, q.Interval, q.Quiet)
if err != nil {
return err
}

switch streams := resp.Data.Result.(type) {
case loghttp.Streams:
if len(streams) == 0 {
return fmt.Errorf("received no log streams")
}
}

return printResult(resp.Data.Result, out)
})
}

func printResult(value loghttp.ResultValue, out output.LogOutput) error {
switch value.Type() {
case logqlmodel.ValueTypeStreams:
Expand Down
4 changes: 2 additions & 2 deletions create/application.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,14 +420,14 @@ func printUnverifiedHostsMessage(app *apps.Application) {
}

func printBuildLogs(ctx context.Context, client *api.Client, build *apps.Build) error {
return client.Log.QueryRange(
return client.Log.QueryRangeWithRetry(
ctx, client.Log.StdOut,
errorLogQuery(logs.BuildQuery(build.Name, build.Namespace)),
)
}

func printReleaseLogs(ctx context.Context, client *api.Client, release *apps.Release) error {
return client.Log.QueryRange(
return client.Log.QueryRangeWithRetry(
ctx, client.Log.StdOut,
errorLogQuery(logs.ApplicationQuery(release.Labels[util.ApplicationNameLabel], release.Namespace)),
)
Expand Down