Merge pull request #49 from KnapsackPro/fallback-mode-in-queue-mode

Fallback mode for Queue Mode when Knapsack Pro API doesn't work

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 * TODO
 
+### 0.48.0
+
+* Fallback mode for Queue Mode when Knapsack Pro API doesn't work.
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/49
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v0.47.0...v0.48.0
+
 ### 0.47.0
 
 * Add in Queue Mode the RSpec summary with info about examples, failures and pending tests.

README.md

@@ -1014,11 +1014,11 @@ There are a few ways to reproduce tests executed on CI node in your development
 
 ##### for knapack_pro regular mode
 
-knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds til it switch to fallback behaviour and it will split test files across CI nodes based on popular test directory names.
+knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds til it switch to fallback behavior and it will split test files across CI nodes based on popular test directory names. When knapack_pro starts fallback mode then you will see a warning in the output.
 
 ##### for knapsack_pro queue mode
 
-knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds til it fails.
+knapack_pro gem will retry requests to Knapsack Pro API multiple times every few seconds till it switches to fallback behavior and it will split test files across CI nodes based on popular test directory names. Note that if one of CI nodes will lose connection to Knapsack Pro API but other not then you may see that some of the test files will be executed on multiple CI nodes. Fallback mode guarantees each of test files is run at least once across CI nodes. Thanks to that we know if the whole test suite is green or not. When knapack_pro starts fallback mode then you will see a warning in the output.
 
 #### How can I change log level?
 

lib/knapsack_pro/allocator.rb

@@ -14,6 +14,7 @@ def test_file_paths
         raise ArgumentError.new(response) if connection.errors?
         prepare_test_files(response)
       else
+        KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. Read more about fallback mode at https://github.com/KnapsackPro/knapsack_pro-ruby#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily")
         fallback_test_files
       end
     end

lib/knapsack_pro/client/connection.rb

@@ -1,7 +1,7 @@
 module KnapsackPro
   module Client
     class Connection
-      TIMEOUT = 30
+      TIMEOUT = 15
       REQUEST_RETRY_TIMEBOX = 2
 
       def initialize(action)
@@ -103,7 +103,7 @@ def post
       rescue Errno::ECONNREFUSED, EOFError, SocketError, Net::OpenTimeout, Net::ReadTimeout => e
         logger.warn(e.inspect)
         retries += 1
-        if retries < 5
+        if retries < 3
           wait = retries * REQUEST_RETRY_TIMEBOX
           logger.warn("Wait #{wait}s and retry request to Knapsack Pro API.")
           sleep wait

lib/knapsack_pro/queue_allocator.rb

@@ -8,15 +8,18 @@ def initialize(args)
       @repository_adapter = args.fetch(:repository_adapter)
     end
 
-    def test_file_paths(can_initialize_queue)
+    def test_file_paths(can_initialize_queue, executed_test_files)
+      return [] if @fallback_activated
       action = build_action(can_initialize_queue)
       connection = KnapsackPro::Client::Connection.new(action)
       response = connection.call
       if connection.success?
         raise ArgumentError.new(response) if connection.errors?
         prepare_test_files(response)
       else
-        raise ArgumentError.new("Couldn't connect with Knapsack Pro API. Response: #{response}")
+        @fallback_activated = true
+        KnapsackPro.logger.warn("Fallback mode started. We could not connect with Knapsack Pro API. Your tests will be executed based on directory names. If other CI nodes were able to connect with Knapsack Pro API then you may notice that some of the test files will be executed twice across CI nodes. The most important thing is to guarantee each of test files is run at least once! Read more about fallback mode at https://github.com/KnapsackPro/knapsack_pro-ruby#what-happens-when-knapsack-pro-api-is-not-availablenot-reachable-temporarily")
+        fallback_test_files(executed_test_files)
       end
     end
 
@@ -52,5 +55,11 @@ def prepare_test_files(response)
       decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(test_files, response['test_files'])
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
+
+    def fallback_test_files(executed_test_files)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_files, ci_node_total)
+      test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
+      KnapsackPro::TestFilePresenter.paths(test_files_for_node_index) - executed_test_files
+    end
   end
 end

lib/knapsack_pro/report.rb

@@ -57,7 +57,9 @@ def self.create_build_subset(test_files)
       response = connection.call
       if connection.success?
         raise ArgumentError.new(response) if connection.errors?
-        KnapsackPro.logger.debug('Saved time execution report on API server.')
+        KnapsackPro.logger.debug('Saved time execution report on Knapsack Pro API server.')
+      else
+        KnapsackPro.logger.warn('Time execution report was not saved on Knapsack Pro API server due to connection problem.')
       end
     end
 

lib/knapsack_pro/runners/queue/base_runner.rb

@@ -17,7 +17,8 @@ def initialize(adapter_class)
 
         def test_file_paths(args)
           can_initialize_queue = args.fetch(:can_initialize_queue)
-          allocator.test_file_paths(can_initialize_queue)
+          executed_test_files = args.fetch(:executed_test_files)
+          allocator.test_file_paths(can_initialize_queue, executed_test_files)
         end
 
         def test_dir

lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -25,7 +25,10 @@ def self.run(args)
         end
 
         def self.run_tests(runner, can_initialize_queue, args, exitstatus, all_test_file_paths)
-          test_file_paths = runner.test_file_paths(can_initialize_queue: can_initialize_queue)
+          test_file_paths = runner.test_file_paths(
+            can_initialize_queue: can_initialize_queue,
+            executed_test_files: all_test_file_paths
+          )
 
           if test_file_paths.empty?
             unless all_test_file_paths.empty?

spec/knapsack_pro/client/connection_spec.rb

@@ -30,8 +30,8 @@
         expect(Net::HTTP).to receive(:new).with('api.knapsackpro.dev', 3000).and_return(http)
 
         expect(http).to receive(:use_ssl=).with(false)
-        expect(http).to receive(:open_timeout=).with(30)
-        expect(http).to receive(:read_timeout=).with(30)
+        expect(http).to receive(:open_timeout=).with(15)
+        expect(http).to receive(:read_timeout=).with(15)
 
         header = { 'X-Request-Id' => 'fake-uuid' }
         http_response = instance_double(Net::HTTPOK, body: body, header: header)

spec/knapsack_pro/queue_allocator_spec.rb

@@ -17,9 +17,10 @@
 
   describe '#test_file_paths' do
     let(:can_initialize_queue) { double }
+    let(:executed_test_files) { [] }
     let(:response) { double }
 
-    subject { queue_allocator.test_file_paths(can_initialize_queue) }
+    subject { queue_allocator.test_file_paths(can_initialize_queue, executed_test_files) }
 
     before do
       encrypted_test_files = double
@@ -80,8 +81,30 @@
       let(:success?) { false }
       let(:errors?) { false }
 
-      it do
-        expect { subject }.to raise_error("Couldn't connect with Knapsack Pro API. Response: #{response}")
+
+      before do
+        test_flat_distributor = instance_double(KnapsackPro::TestFlatDistributor)
+        expect(KnapsackPro::TestFlatDistributor).to receive(:new).with(test_files, ci_node_total).and_return(test_flat_distributor)
+        expect(test_flat_distributor).to receive(:test_files_for_node).with(ci_node_index).and_return([
+          { 'path' => 'c_spec.rb' },
+          { 'path' => 'd_spec.rb' },
+        ])
+      end
+
+      context 'when no test files were executed yet' do
+        let(:executed_test_files) { [] }
+
+        it 'enables fallback mode and returns fallback test files' do
+          expect(subject).to eq ['c_spec.rb', 'd_spec.rb']
+        end
+      end
+
+      context 'when test files were already executed' do
+        let(:executed_test_files) { ['c_spec.rb', 'additional_executed_spec.rb'] }
+
+        it 'enables fallback mode and returns fallback test files' do
+          expect(subject).to eq ['d_spec.rb']
+        end
       end
     end
   end

spec/knapsack_pro/report_spec.rb

@@ -132,7 +132,7 @@
           it do
             logger = instance_double(Logger)
             expect(KnapsackPro).to receive(:logger).and_return(logger)
-            expect(logger).to receive(:debug).with('Saved time execution report on API server.')
+            expect(logger).to receive(:debug).with('Saved time execution report on Knapsack Pro API server.')
             subject
           end
         end
@@ -142,7 +142,12 @@
         let(:success?) { false }
         let(:errors?) { nil }
 
-        it { subject }
+        it do
+          logger = instance_double(Logger)
+          expect(KnapsackPro).to receive(:logger).and_return(logger)
+          expect(logger).to receive(:warn).with('Time execution report was not saved on Knapsack Pro API server due to connection problem.')
+          subject
+        end
       end
     end
 

spec/knapsack_pro/runners/queue/base_runner_spec.rb

@@ -33,11 +33,17 @@
 
       context 'when can_initialize_queue flag has value' do
         let(:can_initialize_queue) { double }
-        let(:args) { { can_initialize_queue: can_initialize_queue } }
+        let(:executed_test_files) { double }
+        let(:args) do
+          {
+            can_initialize_queue: can_initialize_queue,
+            executed_test_files: executed_test_files,
+          }
+        end
         let(:test_file_paths) { double }
 
         before do
-          expect(allocator).to receive(:test_file_paths).and_return(test_file_paths)
+          expect(allocator).to receive(:test_file_paths).with(can_initialize_queue, executed_test_files).and_return(test_file_paths)
         end
 
         it { should eq test_file_paths }

spec/knapsack_pro/runners/queue/rspec_runner_spec.rb

@@ -74,7 +74,7 @@
     subject { described_class.run_tests(runner, can_initialize_queue, args, exitstatus, []) }
 
     before do
-      expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue).and_return(test_file_paths)
+      expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: []).and_return(test_file_paths)
     end
 
     context 'when test files exist' do
@@ -102,7 +102,7 @@
         expect(KnapsackPro::Hooks::Queue).to receive(:call_after_subset_queue)
 
         # second call of run_tests because of recursion
-        expect(runner).to receive(:test_file_paths).with(can_initialize_queue: false).and_return([])
+        expect(runner).to receive(:test_file_paths).with(can_initialize_queue: false, executed_test_files: ['a_spec.rb', 'b_spec.rb']).and_return([])
       end
 
       context 'when exit code is zero' do