Handle connection failure by trying to reconnect and translating the error to ConnectionFailedError so the caller knows to retry

Author: mshearer76

Dockerfile.dev

@@ -2,7 +2,11 @@ FROM ruby:2.4.0
 MAINTAINER data@localytics.com
 
 ENV    DEBIAN_FRONTEND noninteractive
+RUN echo "deb http://deb.debian.org/debian/ jessie main" > /etc/apt/sources.list
+RUN echo "deb-src http://deb.debian.org/debian/ jessie main" >> /etc/apt/sources.list
+RUN echo "deb http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list
+RUN echo "deb-src http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list
 RUN apt-get update && apt-get -y install libnss3-tools unixodbc-dev libmyodbc mysql-client odbc-postgresql  postgresql
 
 WORKDIR /workspace
-CMD docker/docker-entrypoint.sh
+CMD docker/docker-entrypoint.sh

lib/active_record/connection_adapters/odbc_adapter.rb

@@ -76,9 +76,11 @@ class ODBCAdapter < AbstractAdapter
       ADAPTER_NAME = 'ODBC'.freeze
       BOOLEAN_TYPE = 'BOOLEAN'.freeze
 
-      ERR_DUPLICATE_KEY_VALUE     = 23_505
-      ERR_QUERY_TIMED_OUT         = 57_014
-      ERR_QUERY_TIMED_OUT_MESSAGE = /Query has timed out/
+      ERR_DUPLICATE_KEY_VALUE                     = 23_505
+      ERR_QUERY_TIMED_OUT                         = 57_014
+      ERR_QUERY_TIMED_OUT_MESSAGE                 = /Query has timed out/
+      ERR_CONNECTION_FAILED_REGEX                 = '^08[0S]0[12347]'.freeze
+      ERR_CONNECTION_FAILED_MESSAGE               = /Client connection failed/
 
       # The object that stores the information that is fetched from the DBMS
       # when a connection is first established.
@@ -184,6 +186,13 @@ def translate_exception(exception, message)
           ActiveRecord::RecordNotUnique.new(message, exception)
         elsif error_number == ERR_QUERY_TIMED_OUT || exception.message =~ ERR_QUERY_TIMED_OUT_MESSAGE
           ::ODBCAdapter::QueryTimeoutError.new(message, exception)
+        elsif exception.message.match(ERR_CONNECTION_FAILED_REGEX) || exception.message =~ ERR_CONNECTION_FAILED_MESSAGE
+          begin
+            reconnect!
+            ::ODBCAdapter::ConnectionFailedError.new(message, exception)
+          rescue => e
+            puts "unable to reconnect #{e}"
+          end
         else
           super
         end

lib/odbc_adapter/error.rb

@@ -1,4 +1,6 @@
 module ODBCAdapter
   class QueryTimeoutError < ActiveRecord::StatementInvalid
   end
+  class ConnectionFailedError < ActiveRecord::StatementInvalid
+  end
 end

test/connection_fail_test.rb

@@ -0,0 +1,19 @@
+require 'test_helper'
+
+class ConnectionFailTest < Minitest::Test
+  def test_connection_fail
+    # We're only interested in testing a MySQL connection failure for now.
+    # Postgres disconnects generate a different class of errors
+    skip 'Only executed for MySQL' unless ActiveRecord::Base.connection.instance_values['config'][:conn_str].include? 'MySQL'
+    begin
+      conn.execute('KILL CONNECTION_ID();')
+    rescue => e
+      puts "caught exception #{e}"
+    end
+    assert_raises(ODBCAdapter::ConnectionFailedError) { User.average(:letters).round(2) }
+  end
+
+  def conn
+    ActiveRecord::Base.connection
+  end
+end