Swap out pester for Retriable; Retry github errors

.travis.yml

@@ -8,7 +8,10 @@ cache:
   - bundler
   - apt
 rvm:
-  - ruby-2.1.6
+  - 2.1.6
+  - 2.2.8
+  - 2.3.5
+  - 2.4.2
 
 addons:
   postgresql: "9.2"
@@ -29,6 +32,10 @@ env:
     - TEST_SUITE="rspec spec --color --format documentation"
     - TEST_SUITE="rake karma:run"
 
+matrix:
+  allow_failures:
+    - rvm: 2.4.2
+
 before_install:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"

Gemfile

@@ -21,10 +21,10 @@ gem 'foreman'
 gem 'haml'
 gem 'hyperresource'
 gem 'jquery-rails'
+gem 'json', '>= 1.8.5'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-saml', '~> 1.4.1'
 gem 'paper_trail', '~> 4.0.0'
-gem 'pester', '~> 1.0'
 gem 'pg'
 gem 'pg_stream', '~> 0.1.0'
 gem 'puma', '2.11.3'
@@ -34,6 +34,7 @@ gem 'responders', '~> 2.0'
 gem 'resque'
 gem 'resque-pool', '~> 0.5.0'
 gem 'resque-web', '0.0.6', require: 'resque_web'
+gem 'retriable', '~> 3.1'
 gem 'roar'
 gem 'rollbar', '~> 2.3.0'
 gem 'sass-rails'

Gemfile.lock

@@ -161,7 +161,7 @@ GEM
       thor (>= 0.14, < 2.0)
     jquery-ui-rails (5.0.5)
       railties (>= 3.2.16)
-    json (1.8.3)
+    json (1.8.6)
     jwt (1.5.1)
     kaminari (0.16.3)
       actionpack (>= 3.0.0)
@@ -214,7 +214,6 @@ GEM
       activerecord (>= 3.0, < 6.0)
       activesupport (>= 3.0, < 6.0)
       request_store (~> 1.1)
-    pester (1.0.0)
     pg (0.18.3)
     pg_stream (0.1.0)
       pg (>= 0.18.2)
@@ -317,6 +316,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    retriable (3.1.1)
     roar (1.0.3)
       representable (>= 2.0.1, <= 3.0.0)
     rollbar (2.3.0)
@@ -450,11 +450,11 @@ DEPENDENCIES
   haml
   hyperresource
   jquery-rails
+  json (>= 1.8.5)
   launchy
   omniauth-google-oauth2
   omniauth-saml (~> 1.4.1)
   paper_trail (~> 4.0.0)
-  pester (~> 1.0)
   pg
   pg_stream (~> 0.1.0)
   poltergeist
@@ -470,6 +470,7 @@ DEPENDENCIES
   resque
   resque-pool (~> 0.5.0)
   resque-web (= 0.0.6)
+  retriable (~> 3.1)
   roar
   rollbar (~> 2.3.0)
   rspec-rails
@@ -486,4 +487,4 @@ DEPENDENCIES
   zeroclipboard-rails (= 0.1.0)
 
 BUNDLED WITH
-   1.11.2
+   1.15.1

config/initializers/retriable.rb

@@ -0,0 +1,16 @@
+Retriable.configure do |c|
+  c.contexts[:schema_refresh] = {
+    base_interval: 3,
+    tries: 15
+  }
+
+  c.contexts[:s3] = {
+    base_interval: 30,
+    multiplier: 1,
+    tries: 10,
+    on_retry: Proc.new do |exception|
+      # Don't allow retries of misconfigured AWS key errors
+      raise exception if [Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket].include?(exception.class)
+    end
+  }
+end

lib/csv_helper/aws.rb

@@ -4,18 +4,20 @@ class Aws < Base
     S3_FOLDER = APP_CONFIG['s3_folder'] || 'results'
 
     def url
-      Pester.s3.retry do
+      Retriable.with_context(:s3) do
         obj = AwsS3.resource.bucket(S3_BUCKET).object(key)
-        if obj.exists?
-          obj.presigned_url(:get, response_content_disposition: "attachment; filename=result_#{@result_id}.csv", expires_in: 3600)
-        else
-          nil
-        end
+        return nil unless obj.exists?
+
+        obj.presigned_url(
+          :get,
+          response_content_disposition: "attachment; filename=result_#{@result_id}.csv",
+          expires_in: 3600
+        )
       end
     end
 
     def store!
-      Pester.s3.retry do
+      Retriable.with_context(:s3) do
         obj = AwsS3.resource.bucket(S3_BUCKET).object(key)
         obj.upload_file(filepath)
       end

lib/github.rb

@@ -43,7 +43,7 @@ def self.send_request(verb, url, params={})
       request.content_type = 'application/json'
     end
 
-    http.request(request)
+    Retriable.retriable { http.request(request) }
   end
 
   def self.get(url, params={})

lib/schemas/descriptor.rb

@@ -64,12 +64,8 @@ def schema_refresher
 
     def refresh_schema
       Rails.logger.info('Schemas::Descriptor.refresh_schema')
-      result = nil
-      Pester.schema_refresh.retry do
-        result = exec_schema_query
-      end
 
-      if result
+      if (result = Retriable.with_context(:schema_refresh) { exec_schema_query })
         redis_store!(filter_tables(result.to_a))
         @cache = redis_retrieve
       end
@@ -84,7 +80,7 @@ def exec_schema_query
 
     def filter_tables(schemas)
       return schemas unless TABLE_BLACKLIST
-      
+
       schemas.reject do |column|
         schema_blacklist = TABLE_BLACKLIST[column['table_schema']]
         next unless schema_blacklist