Bump Redis timeout for pushing to the queue

Shopify · Sep 12, 2024 · 3f1c3f8 · 3f1c3f8
1 parent 168bd8b
commit 3f1c3f8
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 9 deletions.
diff --git a/ruby/lib/ci/queue/redis/base.rb b/ruby/lib/ci/queue/redis/base.rb
@@ -185,6 +185,14 @@ def max_test_failed?
 
         attr_reader :redis, :redis_url
 
+        def with_redis_timeout(timeout)
+          prev = redis.timeout
+          redis.timeout = timeout
+          yield
+        ensure
+          redis.timeout = prev
+        end
+
         def measure
           starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           yield

diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb
@@ -209,15 +209,17 @@ def push(tests)
 
             attempts = 0
             duration = measure do
-              redis.without_reconnect do
-                redis.multi do |transaction|
-                  transaction.lpush(key('queue'), tests) unless tests.empty?
-                  transaction.set(key('total'), @total)
-                  transaction.set(key('master-status'), 'ready')
-
-                  transaction.expire(key('queue'), config.redis_ttl)
-                  transaction.expire(key('total'), config.redis_ttl)
-                  transaction.expire(key('master-status'), config.redis_ttl)
+              with_redis_timeout(5) do
+                redis.without_reconnect do
+                  redis.multi do |transaction|
+                    transaction.lpush(key('queue'), tests) unless tests.empty?
+                    transaction.set(key('total'), @total)
+                    transaction.set(key('master-status'), 'ready')
+
+                    transaction.expire(key('queue'), config.redis_ttl)
+                    transaction.expire(key('total'), config.redis_ttl)
+                    transaction.expire(key('master-status'), config.redis_ttl)
+                  end
                 end
               rescue StandardError => error
                 if !queue_initialized? && attempts < 3