From 67189b755b9dc8797aa73484b59ea51a12ab1903 Mon Sep 17 00:00:00 2001
From: Sven Fuchs <me@svenfuchs.com>
Date: Mon, 5 Sep 2016 17:58:15 +0200
Subject: [PATCH] retry on redis connection errors

---
 lib/travis/listener/app.rb             |  9 +++++++--
 lib/travis/listener/helper/retrying.rb | 18 ++++++++++++++++++
 spec/travis/helper/retrying_spec.rb    | 13 +++++++++++++
 3 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 lib/travis/listener/helper/retrying.rb
 create mode 100644 spec/travis/helper/retrying_spec.rb

diff --git a/lib/travis/listener/app.rb b/lib/travis/listener/app.rb
index 3ad0fbd..6b8c600 100644
--- a/lib/travis/listener/app.rb
+++ b/lib/travis/listener/app.rb
@@ -2,6 +2,7 @@
 require 'travis/support/logging'
 require 'sidekiq'
 require 'travis/sidekiq/build_request'
+require 'travis/listener/helper/retrying'
 require 'multi_json'
 require 'ipaddr'
 require 'metriks'
@@ -9,7 +10,7 @@
 module Travis
   module Listener
     class App < Sinatra::Base
-      include Logging
+      include Logging, Retrying
 
       # use Rack::CommonLogger for request logging
       enable :logging, :dump_errors
@@ -78,9 +79,13 @@ def valid_ips
 
       def handle_event
         return unless handle_event?
+
         debug "Event payload for #{uuid}: #{payload.inspect}"
         log_event(event_details, uuid: uuid, delivery_guid: delivery_guid, type: event_type, repository: slug)
-        Travis::Sidekiq::BuildRequest.perform_async(data)
+
+        retrying do
+          Travis::Sidekiq::BuildRequest.perform_async(data)
+        end
       end
 
       def handle_event?
diff --git a/lib/travis/listener/helper/retrying.rb b/lib/travis/listener/helper/retrying.rb
new file mode 100644
index 0000000..1d5d82f
--- /dev/null
+++ b/lib/travis/listener/helper/retrying.rb
@@ -0,0 +1,18 @@
+module Travis
+  module Listener
+    module Retrying
+      def retrying(opts = { max: 5, wait: 0.75 })
+        retries ||= 0
+        yield
+      rescue Redis::BaseError => e
+        raise if retries >= opts[:max]
+        retries += 1
+        puts "#{e.class.name}: #{e.message}. Retrying #{retries}/#{opts[:max]}."
+        sleep opts[:wait] * 2 ** (retries - 1) # back off 0.75, 1.5, 3.0, 6.0, 12.0s
+        retry
+      end
+
+      extend self
+    end
+  end
+end
diff --git a/spec/travis/helper/retrying_spec.rb b/spec/travis/helper/retrying_spec.rb
new file mode 100644
index 0000000..8ab6324
--- /dev/null
+++ b/spec/travis/helper/retrying_spec.rb
@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'travis/listener/helper/retrying'
+
+describe Travis::Listener::Retrying do
+  it 'retrying' do
+    count = 0
+    described_class.retrying(max: 5, wait: 0.001) do
+      count += 1
+      raise Redis::BaseError
+    end rescue nil
+    expect(count).to eq 6
+  end
+end