diff --git a/ruby/lib/ci/queue/redis/base.rb b/ruby/lib/ci/queue/redis/base.rb index 750b729d..08bea664 100644 --- a/ruby/lib/ci/queue/redis/base.rb +++ b/ruby/lib/ci/queue/redis/base.rb @@ -185,6 +185,14 @@ def max_test_failed? attr_reader :redis, :redis_url + def with_redis_timeout(timeout) + prev = redis.timeout + redis.timeout = timeout + yield + ensure + redis.timeout = prev + end + def measure starting = Process.clock_gettime(Process::CLOCK_MONOTONIC) yield diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index 52eee420..fff7b3bd 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -209,15 +209,17 @@ def push(tests) attempts = 0 duration = measure do - redis.without_reconnect do - redis.multi do |transaction| - transaction.lpush(key('queue'), tests) unless tests.empty? - transaction.set(key('total'), @total) - transaction.set(key('master-status'), 'ready') - - transaction.expire(key('queue'), config.redis_ttl) - transaction.expire(key('total'), config.redis_ttl) - transaction.expire(key('master-status'), config.redis_ttl) + with_redis_timeout(5) do + redis.without_reconnect do + redis.multi do |transaction| + transaction.lpush(key('queue'), tests) unless tests.empty? + transaction.set(key('total'), @total) + transaction.set(key('master-status'), 'ready') + + transaction.expire(key('queue'), config.redis_ttl) + transaction.expire(key('total'), config.redis_ttl) + transaction.expire(key('master-status'), config.redis_ttl) + end end rescue StandardError => error if !queue_initialized? && attempts < 3