diff --git a/core/config/env/env.go b/core/config/env/env.go index ea84a50b754..37ae131ddf1 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -27,6 +27,8 @@ var ( LOOPPHostName = Var("CL_LOOPP_HOSTNAME") // Work around for Solana LOOPPs configured with zero values. MinOCR2MaxDurationQuery = Var("CL_MIN_OCR2_MAX_DURATION_QUERY") + // PipelineOvertime is an undocumented escape hatch for overriding the default padding in pipeline executions. + PipelineOvertime = Var("CL_PIPELINE_OVERTIME") DatabaseAllowSimplePasswords = Var("CL_DATABASE_ALLOW_SIMPLE_PASSWORDS") DatabaseURL = Secret("CL_DATABASE_URL") diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 895363a53f8..87d4fd8106a 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -236,7 +236,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.11 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 // indirect - github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 // indirect + github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 // indirect github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e // indirect github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 // indirect github.com/smartcontractkit/chainlink-feeds v0.0.0-20231127231053-2232d3a6766d // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index dcf47d392cc..b4a60bd3da6 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1146,8 +1146,8 @@ github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 h1:T3lFWumv github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704/go.mod h1:2QuJdEouTWjh5BDy5o/vgGXQtR4Gz8yH1IYB5eT7u4M= github.com/smartcontractkit/chainlink-automation v1.0.1 h1:vVjBFq2Zsz21kPy1Pb0wpjF9zrbJX+zjXphDeeR4XZk= github.com/smartcontractkit/chainlink-automation v1.0.1/go.mod h1:INSchkV3ntyDdlZKGWA030MPDpp6pbeuiRkRKYFCm2k= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 h1:lSYiaiIfAA+5ac45/UD8ciytlNw/S6fnhK7bxFHYI88= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 h1:7OP1znQwQP3ha1KL5sDjHeKobOfe//YTYdUQH+klkhk= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e h1:xvqffqFec2HkEcUKrCkm4FDJRnn/+gHmvrE/dz3Zlw8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e/go.mod h1:soVgcl4CbfR6hC9UptjuCQhz19HJaFEjwnOpiySkxg0= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 h1:xYqRgZO0nMSO8CBCMR0r3WA+LZ4kNL8a6bnbyk/oBtQ= diff --git a/core/services/pipeline/runner.go b/core/services/pipeline/runner.go index 768f163f9b1..e4d60db881b 100644 --- a/core/services/pipeline/runner.go +++ b/core/services/pipeline/runner.go @@ -15,6 +15,8 @@ import ( "gopkg.in/guregu/null.v4" "github.com/smartcontractkit/chainlink-common/pkg/services" + commonutils "github.com/smartcontractkit/chainlink-common/pkg/utils" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/bridges" "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" @@ -211,18 +213,39 @@ func (r *runner) OnRunFinished(fn func(*Run)) { r.runFinished = fn } -// Be careful with the ctx passed in here: it applies to requests in individual -// tasks but should _not_ apply to the scheduler or run itself +// github.com/smartcontractkit/libocr/offchainreporting2plus/internal/protocol.ReportingPluginTimeoutWarningGracePeriod +var overtime = 100 * time.Millisecond + +func init() { + // undocumented escape hatch + if v := env.PipelineOvertime.Get(); v != "" { + d, err := time.ParseDuration(v) + if err == nil { + overtime = d + } + } +} + func (r *runner) ExecuteRun( ctx context.Context, spec Spec, vars Vars, l logger.Logger, ) (*Run, TaskRunResults, error) { + // Pipeline runs may return results after the context is cancelled, so we modify the + // deadline to give them time to return before the parent context deadline. + var cancel func() + ctx, cancel = commonutils.ContextWithDeadlineFn(ctx, func(orig time.Time) time.Time { + if tenPct := time.Until(orig) / 10; overtime > tenPct { + return orig.Add(-tenPct) + } + return orig.Add(-overtime) + }) + defer cancel() + run := NewRun(spec, vars) pipeline, err := r.initializePipeline(run) - if err != nil { return run, nil, err } diff --git a/go.mod b/go.mod index c71705d9c92..e013a192c08 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/shopspring/decimal v1.3.1 github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 github.com/smartcontractkit/chainlink-automation v1.0.1 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 github.com/smartcontractkit/chainlink-feeds v0.0.0-20231127231053-2232d3a6766d diff --git a/go.sum b/go.sum index b38b7ac2632..35b136e2c9b 100644 --- a/go.sum +++ b/go.sum @@ -1134,8 +1134,8 @@ github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 h1:T3lFWumv github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704/go.mod h1:2QuJdEouTWjh5BDy5o/vgGXQtR4Gz8yH1IYB5eT7u4M= github.com/smartcontractkit/chainlink-automation v1.0.1 h1:vVjBFq2Zsz21kPy1Pb0wpjF9zrbJX+zjXphDeeR4XZk= github.com/smartcontractkit/chainlink-automation v1.0.1/go.mod h1:INSchkV3ntyDdlZKGWA030MPDpp6pbeuiRkRKYFCm2k= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 h1:lSYiaiIfAA+5ac45/UD8ciytlNw/S6fnhK7bxFHYI88= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 h1:7OP1znQwQP3ha1KL5sDjHeKobOfe//YTYdUQH+klkhk= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e h1:xvqffqFec2HkEcUKrCkm4FDJRnn/+gHmvrE/dz3Zlw8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e/go.mod h1:soVgcl4CbfR6hC9UptjuCQhz19HJaFEjwnOpiySkxg0= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 h1:xYqRgZO0nMSO8CBCMR0r3WA+LZ4kNL8a6bnbyk/oBtQ= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 9f0ff1b384c..665fb393052 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -24,7 +24,7 @@ require ( github.com/segmentio/ksuid v1.0.4 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.1 - github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 + github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 github.com/smartcontractkit/chainlink-testing-framework v1.22.0 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index f8a9529a4cc..e3688f68a7f 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1432,8 +1432,8 @@ github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 h1:T3lFWumv github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704/go.mod h1:2QuJdEouTWjh5BDy5o/vgGXQtR4Gz8yH1IYB5eT7u4M= github.com/smartcontractkit/chainlink-automation v1.0.1 h1:vVjBFq2Zsz21kPy1Pb0wpjF9zrbJX+zjXphDeeR4XZk= github.com/smartcontractkit/chainlink-automation v1.0.1/go.mod h1:INSchkV3ntyDdlZKGWA030MPDpp6pbeuiRkRKYFCm2k= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490 h1:lSYiaiIfAA+5ac45/UD8ciytlNw/S6fnhK7bxFHYI88= -github.com/smartcontractkit/chainlink-common v0.1.7-0.20231213134506-b6c433e6c490/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68 h1:7OP1znQwQP3ha1KL5sDjHeKobOfe//YTYdUQH+klkhk= +github.com/smartcontractkit/chainlink-common v0.1.7-0.20231214192257-f53e314deb68/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e h1:xvqffqFec2HkEcUKrCkm4FDJRnn/+gHmvrE/dz3Zlw8= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e/go.mod h1:soVgcl4CbfR6hC9UptjuCQhz19HJaFEjwnOpiySkxg0= github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 h1:xYqRgZO0nMSO8CBCMR0r3WA+LZ4kNL8a6bnbyk/oBtQ=