FFM-7325 - Improve authentication retry logic

lib/ff/ruby/server/sdk/api/auth_service.rb

@@ -2,90 +2,98 @@
 
 class AuthService < Closeable
 
-  def initialize(connector = nil, poll_interval_in_sec = 60, callback = nil, logger = nil)
+  def initialize(connector, callback, logger, retry_delay_ms = 6000)
 
     unless connector.kind_of?(Connector)
-
       raise "The 'connector' parameter must be of '" + Connector.to_s + "' data type"
     end
 
     unless callback.kind_of?(ClientCallback)
-
       raise "The 'callback' parameter must be of '" + ClientCallback.to_s + "' data type"
     end
 
+    @logger = logger
     @callback = callback
     @connector = connector
-    @poll_interval_in_sec = poll_interval_in_sec
-
-    if logger != nil
-
-      @logger = logger
-    else
-
-      @logger = Logger.new(STDOUT)
-    end
+    @retry_delay_ms = retry_delay_ms
+    @authenticated = false
   end
 
   def start_async
-
     @logger.debug "Async starting: " + self.to_s
 
-    @ready = true
-
-    @thread = Thread.new do
-
-      @logger.debug "Async started: " + self.to_s
-
-      while @ready do
-
-        @logger.debug "Async auth iteration"
-
-        if @connector.authenticate
+    @thread = Thread.new :report_on_exception => true do
+      attempt = 1
+      until @authenticated do
+        http_code = @connector.authenticate
 
+        if http_code == 200
+          @authenticated = true
           @callback.on_auth_success
           stop_async
-          @logger.info "Stopping Auth service"
+        elsif should_retry_http_code http_code
+          delay_ms = @retry_delay_ms * [10, attempt].min
+          @logger.warn "Got HTTP code #{http_code} while authenticating on attempt #{attempt}, will retry in #{delay_ms} ms"
+          sleep(delay_ms/1000)
+          attempt += 1
+          @logger.info "Retrying to authenticate, attempt #{attempt}..."
         else
-
-          @logger.error "Exception while authenticating, retry in " + @poll_interval_in_sec.to_s + " seconds"
+          @logger.warn "Auth Service got HTTP code #{http_code} while authenticating, will not attempt to reconnect"
+          @callback.on_auth_failed
+          stop_async
+          next
         end
-
-        sleep(@poll_interval_in_sec)
       end
     end
 
     @thread.run
   end
 
   def close
-
     stop_async
   end
 
-  def on_auth_success
+  protected
 
-    unless @callback == nil
+  def on_auth_success
 
+    if @callback != nil
       unless @callback.kind_of?(ClientCallback)
-
         raise "Expected '" + ClientCallback.to_s + "' data type for the callback"
       end
-
       @callback.on_auth_success
     end
   end
 
-  protected
-
   def stop_async
-
-    @ready = false
-
     if @thread != nil
-
+      @logger.info "Stopping Auth service, status=#{@thread.status}"
       @thread.exit
       @thread = nil
+      @logger.info "Stopping Auth service done"
+    end
+  end
+
+  private
+
+  def is_authenticated
+    @authenticated
+  end
+
+  def should_retry_http_code(code)
+    # 408 request timeout
+    # 425 too early
+    # 429 too many requests
+    # 500 internal server error
+    # 502 bad gateway
+    # 503 service unavailable
+    # 504 gateway timeout
+    #  -1 OpenAPI error (timeout etc)
+    case code
+    when 408,425,429,500,502,503,504,-1
+      return true
+    else
+      return false
     end
   end
 end

lib/ff/ruby/server/sdk/api/client_callback.rb

@@ -2,44 +2,49 @@
 
 class ClientCallback < Closeable
 
+  TBI = RuntimeError.new("To be implemented")
+
   def initialize
     super
-
-    @tbi = "To be implemented"
   end
 
   def on_auth_success
 
-    raise @tbi
+    raise TBI
+  end
+
+  def on_auth_failed
+
+    raise TBI
   end
 
   def on_authorized
 
-    raise @tbi
+    raise TBI
   end
 
   def is_closing
 
-    raise @tbi
+    raise TBI
   end
 
   def on_processor_ready(processor)
 
-    raise @tbi
+    raise TBI
   end
 
   def on_update_processor_ready
 
-    raise @tbi
+    raise TBI
   end
 
   def on_metrics_processor_ready
 
-    raise @tbi
+    raise TBI
   end
 
   def update(message, manual)
 
-    raise @tbi
+    raise TBI
   end
 end

lib/ff/ruby/server/sdk/api/inner_client.rb

@@ -97,17 +97,21 @@ def on_auth_success
     @poll_processor.start
 
     if @config.stream_enabled
-
       @update_processor.start
     end
 
     if @config.analytics_enabled
-
       @metrics_processor.start
     end
 
   end
 
+  def on_auth_failed
+    @config.logger.warn "Authentication failed with a non-recoverable error - defaults will be served"
+    @initialized = true
+
+  end
+
   def close
 
     @config.logger.info "Closing the client: " + self.to_s
@@ -274,9 +278,7 @@ def setup
     end
 
     @auth_service = AuthService.new(
-
       connector = @connector,
-      poll_interval_in_sec = @config.poll_interval_in_seconds,
       callback = self,
       logger = @config.logger
     )
@@ -315,4 +317,5 @@ def synchronize(&block)
 
     @my_mutex.synchronize(&block)
   end
+
 end

lib/ff/ruby/server/sdk/connector/harness_connector.rb

@@ -37,14 +37,21 @@ def authenticate
 
       @config.logger.info "Token has been obtained"
       process_token
-      return true
+      return 200
 
     rescue OpenapiClient::ApiError => e
 
+      if e.message.include? "the server returns an error"
+        # NOTE openapi-generator 5.2.1 has a bug where exceptions don't contain any useful information and we can't
+        # determine if a timeout has occurred. This is fixed in 6.3.0 but requires Ruby version to be increased to 2.7
+        # https://github.com/OpenAPITools/openapi-generator/releases/tag/v6.3.0
+        @config.logger.warn "OpenapiClient::ApiError [\n\n#{e}\n]"
+        return -1
+      end
+
       log_error(e)
+      return e.code
     end
-
-    false
   end
 
   def get_flags
@@ -60,6 +67,7 @@ def get_flags
     rescue OpenapiClient::ApiError => e
 
       log_error(e)
+      return nil
     end
   end
 
@@ -76,6 +84,7 @@ def get_segments
     rescue OpenapiClient::ApiError => e
 
       log_error(e)
+      return nil
     end
   end
 
@@ -174,6 +183,8 @@ def make_api_client
     api_client = OpenapiClient::ApiClient.new
 
     api_client.config = @config
+    api_client.config.connection_timeout = @config.read_timeout / 1000
+    api_client.config.read_timeout = @config.read_timeout / 1000
     api_client.user_agent = @user_agent
     api_client.default_headers['Harness-SDK-Info'] = @sdk_info
 
@@ -239,6 +250,12 @@ def get_query_params
 
   def log_error(e)
 
-    @config.logger.error "ERROR - Start\n\n" + e.to_s + "\nERROR - End"
+    if e.code == 0
+      type = "typhoeus/libcurl"
+    else
+      type = "HTTP code #{e.code}"
+    end
+
+    @config.logger.warn "OpenapiClient::ApiError (#{type}) [\n\n" + e.to_s + "\n]"
   end
 end