From d3568c56f5466467002b2bd4af45591b2b22b96a Mon Sep 17 00:00:00 2001 From: Artur Trzop Date: Fri, 23 Feb 2024 12:35:47 +0100 Subject: [PATCH] Stop calling `RSpec::Core::Runner#run` multiple times in Queue Mode (#237) --------- Co-authored-by: shadre Co-authored-by: Riccardo --- .circleci/config.yml | 89 +- .github/pull_request_template.md | 22 + .gitignore | 4 + CHANGELOG.md | 87 + Gemfile | 9 + README.md | 4 - knapsack_pro.gemspec | 3 +- lib/knapsack_pro.rb | 1 + lib/knapsack_pro/adapters/base_adapter.rb | 9 +- lib/knapsack_pro/adapters/cucumber_adapter.rb | 4 +- lib/knapsack_pro/adapters/rspec_adapter.rb | 25 +- lib/knapsack_pro/config/env.rb | 10 +- .../extensions/rspec_extension.rb | 137 + ...rspec_queue_profile_formatter_extension.rb | 58 - .../rspec_queue_summary_formatter.rb | 145 -- lib/knapsack_pro/formatters/time_tracker.rb | 36 +- .../formatters/time_tracker_fetcher.rb | 6 + lib/knapsack_pro/presenter.rb | 2 +- lib/knapsack_pro/pure/queue/rspec_pure.rb | 92 + lib/knapsack_pro/runners/queue/base_runner.rb | 7 +- .../runners/queue/cucumber_runner.rb | 12 +- .../runners/queue/minitest_runner.rb | 12 +- .../runners/queue/rspec_runner.rb | 297 +-- lib/knapsack_pro/urls.rb | 2 + .../integration/runners/queue/rspec_runner.rb | 80 + .../runners/queue/rspec_runner_spec.rb | 2232 +++++++++++++++++ .../adapters/base_adapter_spec.rb | 28 +- .../adapters/cucumber_adapter_spec.rb | 7 +- .../adapters/rspec_adapter_spec.rb | 26 +- spec/knapsack_pro/config/env_spec.rb | 36 +- .../formatters/time_tracker_specs.rb | 45 +- spec/knapsack_pro/hooks/queue_spec.rb | 4 +- spec/knapsack_pro/presenter_spec.rb | 2 +- .../pure/queue/rspec_pure_spec.rb | 224 ++ .../runners/queue/cucumber_runner_spec.rb | 32 +- .../runners/queue/minitest_runner_spec.rb | 28 +- .../runners/queue/rspec_runner_spec.rb | 536 ---- spec/knapsack_pro_spec.rb | 6 +- 38 files changed, 3217 insertions(+), 1142 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 lib/knapsack_pro/extensions/rspec_extension.rb delete mode 100644 lib/knapsack_pro/formatters/rspec_queue_profile_formatter_extension.rb delete mode 100644 lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb create mode 100644 lib/knapsack_pro/pure/queue/rspec_pure.rb create mode 100644 spec/integration/runners/queue/rspec_runner.rb create mode 100644 spec/integration/runners/queue/rspec_runner_spec.rb create mode 100644 spec/knapsack_pro/pure/queue/rspec_pure_spec.rb delete mode 100644 spec/knapsack_pro/runners/queue/rspec_runner_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index a58a41f2..beb376d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,8 @@ commands: fi - restore_cache: keys: - - v1-bundler-rails-{{ checksum "Gemfile.lock" }}-<< parameters.ruby >> + - v1-bundler-rails-{{ checksum "Gemfile.lock" }}-ruby-<< parameters.ruby >>-rspec-<< parameters.rspec >> + - v1-bundler-rails-{{ checksum "Gemfile.lock" }}-ruby-<< parameters.ruby >>- - v1-bundler-rails-{{ checksum "Gemfile.lock" }}- - v1-bundler-rails- - run: @@ -48,12 +49,13 @@ commands: - save_cache: paths: - << parameters.path >>/vendor/bundle - key: v1-bundler-rails-{{ checksum "Gemfile.lock" }}-<< parameters.ruby >> + key: v1-bundler-rails-{{ checksum "Gemfile.lock" }}-ruby-<< parameters.ruby >>-rspec-<< parameters.rspec >> jobs: unit: parallelism: 1 working_directory: ~/knapsack_pro-ruby + resource_class: small docker: - image: cimg/ruby:3.2 steps: @@ -63,9 +65,51 @@ jobs: - run: bundle exec rspec spec - run: bundle exec ruby spec/knapsack_pro/formatters/time_tracker_specs.rb - integration-regular-rspec: + integration-rspec: + parallelism: 1 + working_directory: ~/knapsack_pro-ruby + resource_class: small + parameters: + ruby: + type: string + rspec: + type: string + docker: + - image: cimg/ruby:<< parameters.ruby >> + steps: + - checkout + - run: + command: | + if [[ "<< parameters.rspec >>" != "" ]]; then + sed -i 's/.*gem.*rspec-core.*/gem "rspec-core", "<< parameters.rspec >>"/g' ./Gemfile + echo "Updated RSpec version in Gemfile" + fi + - restore_cache: + keys: + - v1-bundler-gem-{{ checksum "knapsack_pro.gemspec" }}-ruby-<< parameters.ruby >>-rspec-<< parameters.rspec >> + - v1-bundler-gem-{{ checksum "knapsack_pro.gemspec" }}-ruby-<< parameters.ruby >>- + - v1-bundler-gem-{{ checksum "knapsack_pro.gemspec" }}- + - v1-bundler-gem- + - run: + command: | + bundle config set --local path './vendor/bundle' + bundle install --jobs=4 --retry=3 + - save_cache: + paths: + - ./vendor/bundle + key: v1-bundler-gem-{{ checksum "knapsack_pro.gemspec" }}-ruby-<< parameters.ruby >>-rspec-<< parameters.rspec >> + - run: + command: | + ruby --version + bundle exec rspec --version + RSPEC=$(bundle exec rspec --version | grep rspec-core | head -n1 | cut -d " " -f5) + [ $RSPEC != << parameters.rspec >> ] && exit 1 || echo "Correct version of RSpec installed: $RSPEC" + - run: bundle exec rspec spec/integration/runners/queue/rspec_runner_spec.rb + + e2e-regular-rspec: parallelism: 2 working_directory: ~/knapsack_pro-ruby + resource_class: small parameters: ruby: type: string @@ -122,8 +166,16 @@ jobs: export KNAPSACK_PRO_BRANCH="$CIRCLE_BRANCH--$CIRCLE_BUILD_NUM--regular--split" export KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES=true bundle exec rake knapsack_pro:rspec + - run: + working_directory: ~/rails-app-with-knapsack_pro + command: | + # split custom files by test examples || + export KNAPSACK_PRO_BRANCH="$CIRCLE_BRANCH--$CIRCLE_BUILD_NUM--regular--split-custom-files" + export KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES=true + export KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN="spec/features/calculator_spec.rb" + bundle exec rake knapsack_pro:rspec - integration-queue-rspec: + e2e-queue-rspec: parameters: ruby: type: string @@ -131,6 +183,7 @@ jobs: type: string parallelism: 2 working_directory: ~/knapsack_pro-ruby + resource_class: small docker: - image: cimg/ruby:<< parameters.ruby >>-browsers environment: @@ -216,9 +269,10 @@ jobs: export KNAPSACK_PRO_TEST_FILE_PATTERN="turnip/**/*.feature" bundle exec rake "knapsack_pro:queue:rspec[-r turnip/rspec]" - integration-regular-minitest: + e2e-regular-minitest: parallelism: 2 working_directory: ~/knapsack_pro-ruby + resource_class: small parameters: ruby: type: string @@ -255,12 +309,13 @@ jobs: export KNAPSACK_PRO_BRANCH="$CIRCLE_BRANCH--$CIRCLE_BUILD_NUM--regular" bundle exec rake knapsack_pro:minitest[--verbose] - integration-queue-minitest: + e2e-queue-minitest: parameters: ruby: type: string parallelism: 2 working_directory: ~/knapsack_pro-ruby + resource_class: small docker: - image: cimg/ruby:<< parameters.ruby >>-browsers environment: @@ -307,25 +362,31 @@ workflows: tests: jobs: - unit - - integration-regular-rspec: - name: integration-regular__ruby-<< matrix.ruby >>__rspec-<< matrix.rspec >> + - integration-rspec: + name: integration__ruby-<< matrix.ruby >>__rspec-<< matrix.rspec >> + matrix: + parameters: + ruby: ["3.0", "3.1", "3.2", "3.3"] + rspec: ["3.10.2", "3.11.0", "3.12.2"] + - e2e-regular-rspec: + name: e2e-regular__ruby-<< matrix.ruby >>__rspec-<< matrix.rspec >> matrix: parameters: ruby: ["3.0", "3.1", "3.2", "3.3"] rspec: ["3.10.2", "3.11.0", "3.12.2"] - - integration-queue-rspec: - name: integration-queue__ruby-<< matrix.ruby >>__rspec-<< matrix.rspec >> + - e2e-queue-rspec: + name: e2e-queue__ruby-<< matrix.ruby >>__rspec-<< matrix.rspec >> matrix: parameters: ruby: ["3.0", "3.1", "3.2", "3.3"] rspec: ["3.10.2", "3.11.0", "3.12.2"] - - integration-regular-minitest: - name: integration-regular__ruby-<< matrix.ruby >>__minitest + - e2e-regular-minitest: + name: e2e-regular__ruby-<< matrix.ruby >>__minitest matrix: parameters: ruby: ["3.0", "3.1", "3.2", "3.3"] - - integration-queue-minitest: - name: integration-queue__ruby-<< matrix.ruby >>__minitest + - e2e-queue-minitest: + name: e2e-queue__ruby-<< matrix.ruby >>__minitest matrix: parameters: ruby: ["3.0", "3.1", "3.2", "3.3"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..72332d3f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +# Story + +TODO: link to the internal story + +## Related + +TODO: links to related PRs or issues + +# Description + +TODO + +# Changes + +TODO: changes introduced by this PR + +# Checklist reminder + +- [ ] You follow the architecture outlined below for RSpec in Queue Mode, which is a work in progress (feel free to propose changes): + - Pure: `lib/knapsack_pro/pure/queue/rspec_pure.rb` contains pure functions that are unit tested. + - Extension: `lib/knapsack_pro/extensions/rspec_extension.rb` encapsulates calls to RSpec internals and is integration and e2e tested. + - Runner: `lib/knapsack_pro/runners/queue/rspec_runner.rb` invokes the pure code and the extension to produce side effects, which are integration and e2e tested. diff --git a/.gitignore b/.gitignore index 738703b0..c9bb9321 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,7 @@ Gemfile.lock # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: .rvmrc + +# dynamically generated specs +spec/knapsack_pro/formatters/time_tracker*_spec.rb +spec_integration/ diff --git a/CHANGELOG.md b/CHANGELOG.md index fbba66b4..bc8e8944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,92 @@ # Changelog +### 7.0.0 + +* __(breaking change)__ RSpec in Queue Mode: + * The default for `KNAPSACK_PRO_LOG_LEVEL` is `info` instead of `debug`. + * The RSpec `before(:suite)` and `after(:suite)` hooks changed: + + __Before:__
+ The `before(:suite)` and `after(:suite)` hooks were executed multiple times. Each time for a batch of tests fetched from Knapsack Pro Queue API. + + __After:__
+ The `before(:suite)` and `after(:suite)` hooks are executed only once: `before(:suite)` is executed before starting tests, `after(:suite)` is executed after all tests are completed. (It is what you would expect from RSpec). + + * The `KnapsackPro::Hooks::Queue.after_queue` hook change: + + __Before:__
+ The `KnapsackPro::Hooks::Queue.after_queue` hook is executed outside of the `after(:suite)` hook. + + __After:__
+ The `KnapsackPro::Hooks::Queue.after_queue` hook is executed __inside__ of the `after(:suite)` hook. + +* Recommended RSpec changes in your project: + * Remove the following code if you use Queue Mode and the `rspec_junit_formatter` gem to generate JUnit XML or JSON reports: + + ```ruby + # REMOVE THE FOLLOWING CODE + + # spec_helper.rb or rails_helper.rb + TMP_REPORT = "tmp/rspec_#{ENV['KNAPSACK_PRO_CI_NODE_INDEX']}.xml" + FINAL_REPORT = "tmp/final_rspec_#{ENV['KNAPSACK_PRO_CI_NODE_INDEX']}.xml" + + KnapsackPro::Hooks::Queue.after_subset_queue do |queue_id, subset_queue_id| + if File.exist?(TMP_REPORT) + FileUtils.mv(TMP_REPORT, FINAL_REPORT) + end + end + ``` + + Learn more about [using Knapsack Pro with RSpec formatters](https://docs.knapsackpro.com/ruby/rspec/#formatters-rspec_junit_formatter-json) and [using Knapsack Pro with CircleCI](https://docs.knapsackpro.com/ruby/circleci/) in the docs. + + * Replace the following code if you are using Queue Mode and the `percy-capybara` gem on a version older than 4: + + Before: + + ```ruby + KnapsackPro::Hooks::Queue.before_queue { |queue_id| Percy::Capybara.initialize_build } + KnapsackPro::Hooks::Queue.after_queue { |queue_id| Percy::Capybara.finalize_build } + ``` + + After: + + ```ruby + # recommended + before(:suite) { Percy::Capybara.initialize_build } + after(:suite) { Percy::Capybara.finalize_build } + ``` + + Learn more about [using Knapsack Pro with Percy](https://docs.knapsackpro.com/ruby/hooks/#percy-capybara) in the docs. + + * We are no longer modifying the default RSpec formatters in Queue Mode. You can remove the [`KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS`](https://docs.knapsackpro.com/ruby/reference/#knapsack_pro_modify_default_rspec_formatters-removed-rspec) environment variable from your CI config if you are using it. + +* RSpec improvements in Queue Mode: + * Termination signals (`HUP`, `INT`, `TERM`, `ABRT`, `QUIT`, `USR1`, and `USR2`) are handled earlier: the process will terminate before the next top-level example group (`describe` or `context`) instead of waiting for the next Knapsack Pro batch of tests. + + * Respect the `--error-exit-code` option. It sets a custom exit code (instead of `1`) when RSpec fails outside an example (e.g. lack of memory, termination signal). + + ```bash + bundle exec rake "knapsack_pro:queue:rspec[--error-exit-code 3]" + ``` + + * Respect the `--failure-exit-code` option. It sets a custom exit code for when any examples fail. + + ```bash + bundle exec rake "knapsack_pro:queue:rspec[--failure-exit-code 2]" + ``` + + * Respect the `--fail-fast` option and show a warning in the Knapsack Pro log. + + * Ignore the `fail_if_no_examples` option in Queue Mode: + * A late CI node, started after all tests were executed by other nodes, is expected to receive an empty batch. + * A batch could contain tests with no examples (e.g. commented out) + + * Raise an exception if the [deprecated `run_all_when_everything_filtered`](https://docs.knapsackpro.com/ruby/rspec/#some-of-my-test-files-are-not-executed) option is detected. + +PR with the above changes: https://github.com/KnapsackPro/knapsack_pro-ruby/pull/237 + +https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v6.0.4...v7.0.0 + ### 6.0.4 * fix(minitest): avoid installing `at_exit` (that would result in an empty run of Minitest after Knapsack Pro is finished in Queue Mode) diff --git a/Gemfile b/Gemfile index 91a87a51..38e95fc7 100644 --- a/Gemfile +++ b/Gemfile @@ -2,3 +2,12 @@ source 'https://rubygems.org' # Specify your gem's dependencies in knapsack.gemspec gemspec + +group :test do + gem 'rspec_junit_formatter', require: false + gem 'nokogiri', require: false + gem 'simplecov', require: false + + # This line is going to be replaced on CI to test different RSpec versions. + # gem 'rspec-core', 'x.x.x' +end diff --git a/README.md b/README.md index 5d38fde1..62a19fcb 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,6 @@ The `knapsack_pro` gem supports all CIs and the following test runners: - Spinach - Turnip -## Requirements - -`>= Ruby 2.1.0` - ## Installation The [Installation Guide](https://docs.knapsackpro.com/knapsack_pro-ruby/guide/?utm_source=github&utm_medium=readme&utm_campaign=knapsack_pro-ruby_gem&utm_content=installation_guide) will ask you a few questions and generate instruction steps for your project: diff --git a/knapsack_pro.gemspec b/knapsack_pro.gemspec index 815774ba..6ac4b5ef 100644 --- a/knapsack_pro.gemspec +++ b/knapsack_pro.gemspec @@ -6,10 +6,11 @@ require 'knapsack_pro/version' Gem::Specification.new do |spec| spec.name = 'knapsack_pro' spec.version = KnapsackPro::VERSION + spec.required_ruby_version = '>= 2.7.0' spec.authors = ['ArturT'] spec.email = ['support@knapsackpro.com'] spec.summary = %q{Knapsack Pro splits tests across parallel CI nodes and ensures each parallel job finish work at a similar time.} - spec.description = %q{Run tests in parallel across CI server nodes based on tests execution time. Split tests in a dynamic way to ensure parallel jobs are done at a similar time. Thanks to that your CI build time is as fast as possible. It works with many CI providers.} + spec.description = %q{Knapsack Pro wraps your current test runner(s) and works with your existing CI infrastructure to parallelize tests optimally. It dynamically splits your tests based on up-to-date test execution data. It's designed from the ground up for CI and supports all of them.} spec.homepage = 'https://knapsackpro.com' spec.license = 'MIT' spec.metadata = { diff --git a/lib/knapsack_pro.rb b/lib/knapsack_pro.rb index f6081397..12d9e2cc 100644 --- a/lib/knapsack_pro.rb +++ b/lib/knapsack_pro.rb @@ -82,6 +82,7 @@ require_relative 'knapsack_pro/crypto/branch_encryptor' require_relative 'knapsack_pro/crypto/decryptor' require_relative 'knapsack_pro/crypto/digestor' +require_relative 'knapsack_pro/pure/queue/rspec_pure' require 'knapsack_pro/railtie' if defined?(Rails::Railtie) diff --git a/lib/knapsack_pro/adapters/base_adapter.rb b/lib/knapsack_pro/adapters/base_adapter.rb index 983c63bb..a47d4448 100644 --- a/lib/knapsack_pro/adapters/base_adapter.rb +++ b/lib/knapsack_pro/adapters/base_adapter.rb @@ -60,13 +60,13 @@ def bind File.write(self.class.adapter_bind_method_called_file, nil) if KnapsackPro::Config::Env.recording_enabled? - KnapsackPro.logger.debug('Test suite time execution recording enabled.') + KnapsackPro.logger.debug('Regular Mode enabled.') bind_time_tracker bind_save_report end if KnapsackPro::Config::Env.queue_recording_enabled? - KnapsackPro.logger.debug('Test suite time execution queue recording enabled.') + KnapsackPro.logger.debug('Queue Mode enabled.') bind_queue_mode end end @@ -83,8 +83,13 @@ def bind_before_queue_hook raise NotImplementedError end + def bind_after_queue_hook + raise NotImplementedError + end + def bind_queue_mode bind_before_queue_hook + bind_after_queue_hook bind_time_tracker end end diff --git a/lib/knapsack_pro/adapters/cucumber_adapter.rb b/lib/knapsack_pro/adapters/cucumber_adapter.rb index ac512e31..8f05791e 100644 --- a/lib/knapsack_pro/adapters/cucumber_adapter.rb +++ b/lib/knapsack_pro/adapters/cucumber_adapter.rb @@ -65,9 +65,7 @@ def bind_before_queue_hook end end - def bind_queue_mode - super - + def bind_after_queue_hook ::Kernel.at_exit do KnapsackPro::Hooks::Queue.call_after_subset_queue KnapsackPro::Report.save_subset_queue_to_file diff --git a/lib/knapsack_pro/adapters/rspec_adapter.rb b/lib/knapsack_pro/adapters/rspec_adapter.rb index 1bbc3c73..e274af1d 100644 --- a/lib/knapsack_pro/adapters/rspec_adapter.rb +++ b/lib/knapsack_pro/adapters/rspec_adapter.rb @@ -87,7 +87,7 @@ def self.top_level_group(example) def bind_time_tracker ensure_no_focus! - log_batch_duration + log_tests_duration end def ensure_no_focus! @@ -105,12 +105,14 @@ def ensure_no_focus! end end - def log_batch_duration + def log_tests_duration ::RSpec.configure do |config| - config.after(:suite) do + config.append_after(:suite) do time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call - formatted = KnapsackPro::Presenter.global_time(time_tracker.batch_duration) - KnapsackPro.logger.debug(formatted) + if time_tracker + formatted = KnapsackPro::Presenter.global_time(time_tracker.duration) + KnapsackPro.logger.debug(formatted) + end end end end @@ -127,10 +129,15 @@ def bind_save_report def bind_before_queue_hook ::RSpec.configure do |config| config.before(:suite) do - unless ENV['KNAPSACK_PRO_BEFORE_QUEUE_HOOK_CALLED'] - ENV['KNAPSACK_PRO_BEFORE_QUEUE_HOOK_CALLED'] = 'true' - KnapsackPro::Hooks::Queue.call_before_queue - end + KnapsackPro::Hooks::Queue.call_before_queue + end + end + end + + def bind_after_queue_hook + ::RSpec.configure do |config| + config.after(:suite) do + KnapsackPro::Hooks::Queue.call_after_queue end end end diff --git a/lib/knapsack_pro/config/env.rb b/lib/knapsack_pro/config/env.rb index 508e6029..eeeecb38 100644 --- a/lib/knapsack_pro/config/env.rb +++ b/lib/knapsack_pro/config/env.rb @@ -144,14 +144,6 @@ def test_files_encrypted? test_files_encrypted == 'true' end - def modify_default_rspec_formatters - ENV.fetch('KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS', true) - end - - def modify_default_rspec_formatters? - modify_default_rspec_formatters.to_s == 'true' - end - def branch_encrypted ENV['KNAPSACK_PRO_BRANCH_ENCRYPTED'] end @@ -276,7 +268,7 @@ def ci_provider end def log_level - LOG_LEVELS[ENV['KNAPSACK_PRO_LOG_LEVEL'].to_s.downcase] || ::Logger::DEBUG + LOG_LEVELS[ENV['KNAPSACK_PRO_LOG_LEVEL'].to_s.downcase] || ::Logger::INFO end def log_dir diff --git a/lib/knapsack_pro/extensions/rspec_extension.rb b/lib/knapsack_pro/extensions/rspec_extension.rb new file mode 100644 index 00000000..ce42400b --- /dev/null +++ b/lib/knapsack_pro/extensions/rspec_extension.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module KnapsackPro + module Extensions + # Facade to abstract calls to internal RSpec methods. + # To allow comparing the monkey patch with the original RSpec code, keep a similar method structure and permalink to the source. + module RSpecExtension + Seed = Struct.new(:value, :used?) + + def self.setup! + RSpec::Core::World.prepend(World) + RSpec::Core::Runner.prepend(Runner) + RSpec::Core::Configuration.prepend(Configuration) + end + + module World + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/world.rb#L171 + # + # Filters are not announced because we do not load tests during setup. It would print `No examples found.` and we don't want that. + def knapsack__announce_filters + fail_if_config_and_cli_options_invalid + end + end + + module Runner + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/runner.rb#L98 + # + # `@configuration.load_spec_files` is not called because we load tests in batches with `knapsack__load_spec_files_batch` later on. + def knapsack__setup(stream_error = $stderr, stream_out = $stdout) + configure(stream_error, stream_out) + ensure + world.knapsack__announce_filters + end + + def knapsack__wants_to_quit? + world.wants_to_quit + end + + def knapsack__rspec_is_quitting? + world.respond_to?(:rspec_is_quitting) && world.rspec_is_quitting + end + + def knapsack__exit_early + _exit_status = configuration.reporter.exit_early(exit_code) + end + + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/configuration.rb#L546 + def knapsack__error_exit_code + configuration.error_exit_code # nil unless `--error-exit-code` is specified + end + + # must be called after `Runner#knapsack__setup` that loads the `spec_helper.rb` configuration + def knapsack__deprecated_run_all_when_everything_filtered_enabled? + configuration.respond_to?(:run_all_when_everything_filtered) && !!configuration.run_all_when_everything_filtered + end + + def knapsack__seed + Seed.new(configuration.seed.to_s, configuration.seed_used?) + end + + # @param test_file_paths Array[String] + # Example: ['a_spec.rb', 'b_spec.rb[1:1]'] + def knapsack__load_spec_files_batch(test_file_paths) + world.reset + + # Reset filters, but do not reset `configuration.static_config_filter_manager` to preserve the --tag option + filter_manager = RSpec::Core::FilterManager.new + options.configure_filter_manager(filter_manager) + configuration.filter_manager = filter_manager + + configuration.knapsack__load_spec_files(test_file_paths) + end + + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/runner.rb#L113 + # + # Ignore `configuration.fail_if_no_examples` in Queue Mode: + # * a late CI node, started after all tests were executed by other nodes, is expected to receive an empty batch + # * a batch could contain tests with no examples (e.g. commented out) + # + # @return [Fixnum] exit status code. + def knapsack__run_specs(queue_runner) + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/world.rb#L53 + ordering_strategy = configuration.ordering_registry.fetch(:global) + node_examples_passed = true + + configuration.reporter.report(_expected_example_count = 0) do |reporter| + configuration.with_suite_hooks do + queue_runner.with_batch do |test_file_paths| + knapsack__load_spec_files_batch(test_file_paths) + + examples_passed = ordering_strategy.order(world.example_groups).map do |example_group| + queue_runner.handle_signal! + example_group.run(reporter) + end.all? + + node_examples_passed = false unless examples_passed + + knapsack__persist_example_statuses + + if reporter.fail_fast_limit_met? + queue_runner.log_fail_fast_limit_met + break + end + end + end + + exit_code(node_examples_passed) + end + end + + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/runner.rb#L90 + def knapsack__persist_example_statuses + persist_example_statuses + end + end + + module Configuration + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/configuration.rb#L1619 + def knapsack__load_spec_files(test_file_paths) + batch_of_files_to_run = get_files_to_run(test_file_paths) + batch_of_files_to_run.each do |f| + file = File.expand_path(f) + load_file_handling_errors(:load, file) + loaded_spec_files << file + end + end + end + end + end +end diff --git a/lib/knapsack_pro/formatters/rspec_queue_profile_formatter_extension.rb b/lib/knapsack_pro/formatters/rspec_queue_profile_formatter_extension.rb deleted file mode 100644 index d0e3e9ee..00000000 --- a/lib/knapsack_pro/formatters/rspec_queue_profile_formatter_extension.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -RSpec::Support.require_rspec_core('formatters/profile_formatter') - -module KnapsackPro - module Formatters - module RSpecQueueProfileFormatterExtension - def self.print_summary - return unless KnapsackPro::Config::Env.modify_default_rspec_formatters? - ::RSpec::Core::Formatters::ProfileFormatter.print_profile_summary - end - - def initialize(output) - @output = output - self.class.registered_output = output - end - - def dump_profile(profile) - self.class.most_recent_profile = profile - end - end - end -end - -if KnapsackPro::Config::Env.modify_default_rspec_formatters? - class RSpec::Core::Formatters::ProfileFormatter - prepend KnapsackPro::Formatters::RSpecQueueProfileFormatterExtension - - def self.registered_output=(output) - @registered_output = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => output - } - end - - def self.registered_output - @registered_output ||= {} - @registered_output[ENV['KNAPSACK_PRO_QUEUE_ID']] - end - - def self.most_recent_profile=(profile) - @most_recent_profile = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => profile - } - end - - def self.most_recent_profile - @most_recent_profile ||= {} - @most_recent_profile[ENV['KNAPSACK_PRO_QUEUE_ID']] || [] - end - - def self.print_profile_summary - return unless registered_output - profile_formatter = new(registered_output) - profile_formatter.send(:dump_profile_slowest_examples, most_recent_profile) - profile_formatter.send(:dump_profile_slowest_example_groups, most_recent_profile) - end - end -end diff --git a/lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb b/lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb deleted file mode 100644 index 4d68b4d3..00000000 --- a/lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true - -require_relative './time_tracker_fetcher' - -RSpec::Support.require_rspec_core('formatters/base_formatter') -RSpec::Support.require_rspec_core('formatters/base_text_formatter') - -module KnapsackPro - module Formatters - module RSpecHideFailuresAndPendingExtension - def dump_failures(notification); end - def dump_pending(notification); end - def dump_summary(summary); end - end - - class RSpecQueueSummaryFormatter < ::RSpec::Core::Formatters::BaseFormatter - ::RSpec::Core::Formatters.register self, :dump_summary, :dump_failures, :dump_pending - - def self.registered_output=(output) - @registered_output = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => output - } - end - - def self.registered_output - @registered_output[ENV['KNAPSACK_PRO_QUEUE_ID']] - end - - def self.most_recent_failures_summary=(fully_formatted_failed_examples) - @most_recent_failures_summary = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted_failed_examples - } - end - - def self.most_recent_failures_summary - @most_recent_failures_summary ||= {} - @most_recent_failures_summary[ENV['KNAPSACK_PRO_QUEUE_ID']] || [] - end - - def self.most_recent_pending=(fully_formatted_pending_examples) - @most_recent_pending = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted_pending_examples - } - end - - def self.most_recent_pending - @most_recent_pending ||= {} - @most_recent_pending[ENV['KNAPSACK_PRO_QUEUE_ID']] || [] - end - - def self.most_recent_summary=(fully_formatted) - @most_recent_summary = { - ENV['KNAPSACK_PRO_QUEUE_ID'] => fully_formatted - } - end - - def self.most_recent_summary - @most_recent_summary ||= {} - @most_recent_summary[ENV['KNAPSACK_PRO_QUEUE_ID']] || [] - end - - def self.print_summary - registered_output.puts('Knapsack Pro Queue finished!') - registered_output.puts('') - - unless most_recent_pending.empty? - registered_output.puts('All pending tests on this CI node:') - registered_output.puts(most_recent_pending) - registered_output.puts('') - end - - unless most_recent_failures_summary.empty? - registered_output.puts('All failed tests on this CI node:') - registered_output.puts(most_recent_failures_summary) - registered_output.puts('') - end - - registered_output.puts(most_recent_summary) - end - - def self.print_exit_summary(all_test_file_paths) - registered_output.puts('Knapsack Pro Queue exited/aborted!') - registered_output.puts('') - - time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call - unexecuted_test_files = time_tracker&.unexecuted_test_files(all_test_file_paths) || [] - unless unexecuted_test_files.empty? - registered_output.puts('Unexecuted tests on this CI node:') - registered_output.puts(unexecuted_test_files) - registered_output.puts('') - end - - unless most_recent_pending.empty? - registered_output.puts('All pending tests on this CI node:') - registered_output.puts(most_recent_pending) - registered_output.puts('') - end - - unless most_recent_failures_summary.empty? - registered_output.puts('All failed tests on this CI node:') - registered_output.puts(most_recent_failures_summary) - registered_output.puts('') - end - - registered_output.puts(most_recent_summary) - end - - def initialize(output) - super - self.class.registered_output = output - end - - def dump_failures(notification) - return if notification.failure_notifications.empty? - self.class.most_recent_failures_summary = notification.fully_formatted_failed_examples - end - - def dump_pending(notification) - return if notification.pending_examples.empty? - self.class.most_recent_pending = notification.fully_formatted_pending_examples - end - - def dump_summary(summary) - colorizer = ::RSpec::Core::Formatters::ConsoleCodes - duration = KnapsackPro::Formatters::TimeTrackerFetcher.call.duration - formatted_duration = ::RSpec::Core::Formatters::Helpers.format_duration(duration) - - formatted = "\nFinished in #{formatted_duration}\n" \ - "#{summary.colorized_totals_line(colorizer)}\n" - - unless summary.failed_examples.empty? - formatted += (summary.colorized_rerun_commands(colorizer) + "\n") - end - - self.class.most_recent_summary = formatted - end - end - end -end - -if KnapsackPro::Config::Env.modify_default_rspec_formatters? - class RSpec::Core::Formatters::BaseTextFormatter - prepend KnapsackPro::Formatters::RSpecHideFailuresAndPendingExtension - end -end diff --git a/lib/knapsack_pro/formatters/time_tracker.rb b/lib/knapsack_pro/formatters/time_tracker.rb index 4e2637a2..289c0d83 100644 --- a/lib/knapsack_pro/formatters/time_tracker.rb +++ b/lib/knapsack_pro/formatters/time_tracker.rb @@ -9,24 +9,18 @@ class TimeTracker :example_group_started, :example_started, :example_finished, - :example_group_finished, - :stop + :example_group_finished attr_reader :output # RSpec < v3.10.2 - # Called at the beginning of each batch, - # but only the first instance of this class is used, - # so don't rely on the initializer to reset values. def initialize(_output) @output = StringIO.new @time_each = nil @time_all = nil @before_all = 0.0 @group = {} - @batch = {} - @queue = {} + @paths = {} @suite_started = now - @batch_started = now end def example_group_started(notification) @@ -47,21 +41,15 @@ def example_finished(notification) def example_group_finished(notification) return unless top_level_group?(notification.group) - add_hooks_time(@group, @before_all, now - @time_all) - @batch = merge(@batch, @group) + after_all = @time_all.nil? ? 0.0 : now - @time_all + add_hooks_time(@group, @before_all, after_all) @before_all = 0.0 + @paths = merge(@paths, @group) @group = {} end - # Called at the end of each batch - def stop(_notification) - @queue = merge(@queue, @batch) - @batch = {} - @batch_started = now - end - def queue(scheduled_paths) - recorded_paths = @queue.values.map do |example| + recorded_paths = @paths.values.map do |example| KnapsackPro::Adapters::RSpecAdapter.parse_file_path(example[:path]) end @@ -69,13 +57,13 @@ def queue(scheduled_paths) object[path] = { path: path, time_execution: 0.0 } end - merge(@queue, missing).values.map do |example| + merge(@paths, missing).values.map do |example| example.transform_keys(&:to_s) end end def batch - @batch.values.map do |example| + @paths.values.map do |example| example.transform_keys(&:to_s) end end @@ -84,17 +72,13 @@ def duration now - @suite_started end - def batch_duration - now - @batch_started - end - def unexecuted_test_files(scheduled_paths) - pending_paths = (@queue.values + @batch.values) + pending_paths = @paths.values .filter { |example| example[:time_execution] == 0.0 } .map { |example| example[:path] } not_run_paths = scheduled_paths - - (@queue.values + @batch.values) + @paths.values .map { |example| example[:path] } pending_paths + not_run_paths diff --git a/lib/knapsack_pro/formatters/time_tracker_fetcher.rb b/lib/knapsack_pro/formatters/time_tracker_fetcher.rb index 84034387..644dfd5c 100644 --- a/lib/knapsack_pro/formatters/time_tracker_fetcher.rb +++ b/lib/knapsack_pro/formatters/time_tracker_fetcher.rb @@ -9,6 +9,12 @@ def self.call .formatters .find { |f| f.class.to_s == "KnapsackPro::Formatters::TimeTracker" } end + + def self.unexecuted_test_files(scheduled_paths) + time_tracker = call + return [] unless time_tracker + time_tracker.unexecuted_test_files(scheduled_paths) + end end end end diff --git a/lib/knapsack_pro/presenter.rb b/lib/knapsack_pro/presenter.rb index 495de2ba..6db1f972 100644 --- a/lib/knapsack_pro/presenter.rb +++ b/lib/knapsack_pro/presenter.rb @@ -6,7 +6,7 @@ class << self def global_time(time = nil) time = KnapsackPro.tracker.global_time if time.nil? global_time = pretty_seconds(time) - "Global time execution for tests: #{global_time}" + "Global test execution duration: #{global_time}" end def pretty_seconds(seconds) diff --git a/lib/knapsack_pro/pure/queue/rspec_pure.rb b/lib/knapsack_pro/pure/queue/rspec_pure.rb new file mode 100644 index 00000000..94fd55f3 --- /dev/null +++ b/lib/knapsack_pro/pure/queue/rspec_pure.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module KnapsackPro + module Pure + module Queue + class RSpecPure + FAILURE_EXIT_CODE = 1 + FORMATTERS = [ + 'KnapsackPro::Formatters::TimeTracker', + ] + + def add_knapsack_pro_formatters_to(spec_opts) + return spec_opts unless spec_opts + return spec_opts if FORMATTERS.all? { |formatter| spec_opts.include?(formatter) } + + FORMATTERS.each do |formatter| + next if spec_opts.include?(formatter) + spec_opts += " --format #{formatter}" + end + + spec_opts + end + + def error_exit_code(rspec_error_exit_code) + rspec_error_exit_code || FAILURE_EXIT_CODE + end + + def args_with_seed_option_added_when_viable(order_option, seed, args) + return args if order_option && !order_option.include?('rand') + return args if order_option && order_option.to_s.split(':')[1] + + return args unless seed.used? + + args + ['--seed', seed.value] + end + + def prepare_cli_args(args, has_format_option, test_dir) + (args || '').split + .yield_self { args_with_at_least_one_formatter(_1, has_format_option) } + .yield_self { args_with_default_options(_1, test_dir) } + end + + def rspec_command(args, test_file_paths, scope) + messages = [] + return messages if test_file_paths.empty? + + case scope + when :batch_finished + messages << 'To retry the last batch of tests fetched from the Queue API, please run the following command on your machine:' + when :queue_finished + messages << 'To retry all the tests assigned to this CI node, please run the following command on your machine:' + end + + stringified_cli_args = args.join(' ') + FORMATTERS.each do |formatter| + stringified_cli_args.sub!(" --format #{formatter}", '') + end + + messages << "bundle exec rspec #{stringified_cli_args} " + KnapsackPro::TestFilePresenter.stringify_paths(test_file_paths) + + messages + end + + def exit_summary(unexecuted_test_files) + return if unexecuted_test_files.empty? + + "Unexecuted tests on this CI node (including pending tests): #{unexecuted_test_files.join(' ')}" + end + + private + + def args_with_at_least_one_formatter(cli_args, has_format_option) + return cli_args if has_format_option + + cli_args + ['--format', 'progress'] + end + + def args_with_default_options(cli_args, test_dir) + new_cli_args = cli_args + [ + '--default-path', test_dir, + ] + + FORMATTERS.each do |formatter| + new_cli_args += ['--format', formatter] + end + + new_cli_args + end + end + end + end +end diff --git a/lib/knapsack_pro/runners/queue/base_runner.rb b/lib/knapsack_pro/runners/queue/base_runner.rb index e748b87b..e925d68c 100644 --- a/lib/knapsack_pro/runners/queue/base_runner.rb +++ b/lib/knapsack_pro/runners/queue/base_runner.rb @@ -4,6 +4,7 @@ module KnapsackPro module Runners module Queue class BaseRunner + TerminationError = Class.new(StandardError) TERMINATION_SIGNALS = %w(HUP INT TERM ABRT QUIT USR1 USR2) @@terminate_process = false @@ -42,13 +43,17 @@ def self.child_status end def self.handle_signal! - raise 'Knapsack Pro process was terminated!' if @@terminate_process + raise TerminationError.new('Knapsack Pro process was terminated!') if @@terminate_process end def self.set_terminate_process @@terminate_process = true end + def set_terminate_process + self.class.set_terminate_process + end + def trap_signals TERMINATION_SIGNALS.each do |signal| Signal.trap(signal) { diff --git a/lib/knapsack_pro/runners/queue/cucumber_runner.rb b/lib/knapsack_pro/runners/queue/cucumber_runner.rb index 3cbdb469..fc4909d4 100644 --- a/lib/knapsack_pro/runners/queue/cucumber_runner.rb +++ b/lib/knapsack_pro/runners/queue/cucumber_runner.rb @@ -21,7 +21,7 @@ def self.run(args) can_initialize_queue: true, args: args, exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } while accumulator[:status] == :next handle_signal! @@ -36,15 +36,15 @@ def self.run_tests(accumulator) can_initialize_queue = accumulator.fetch(:can_initialize_queue) args = accumulator.fetch(:args) exitstatus = accumulator.fetch(:exitstatus) - all_test_file_paths = accumulator.fetch(:all_test_file_paths) + node_test_file_paths = accumulator.fetch(:node_test_file_paths) test_file_paths = runner.test_file_paths( can_initialize_queue: can_initialize_queue, - executed_test_files: all_test_file_paths + executed_test_files: node_test_file_paths ) if test_file_paths.empty? - unless all_test_file_paths.empty? + unless node_test_file_paths.empty? KnapsackPro::Adapters::CucumberAdapter.verify_bind_method_called end @@ -65,7 +65,7 @@ def self.run_tests(accumulator) KnapsackPro::Hooks::Queue.call_before_subset_queue - all_test_file_paths += test_file_paths + node_test_file_paths += test_file_paths result_exitstatus = cucumber_run(runner, test_file_paths, args) exitstatus = result_exitstatus if result_exitstatus != 0 @@ -80,7 +80,7 @@ def self.run_tests(accumulator) can_initialize_queue: false, args: args, exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, + node_test_file_paths: node_test_file_paths, } end end diff --git a/lib/knapsack_pro/runners/queue/minitest_runner.rb b/lib/knapsack_pro/runners/queue/minitest_runner.rb index cf66db52..31778928 100644 --- a/lib/knapsack_pro/runners/queue/minitest_runner.rb +++ b/lib/knapsack_pro/runners/queue/minitest_runner.rb @@ -32,7 +32,7 @@ def self.run(args) can_initialize_queue: true, args: cli_args, exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } while accumulator[:status] == :next handle_signal! @@ -47,15 +47,15 @@ def self.run_tests(accumulator) can_initialize_queue = accumulator.fetch(:can_initialize_queue) args = accumulator.fetch(:args) exitstatus = accumulator.fetch(:exitstatus) - all_test_file_paths = accumulator.fetch(:all_test_file_paths) + node_test_file_paths = accumulator.fetch(:node_test_file_paths) test_file_paths = runner.test_file_paths( can_initialize_queue: can_initialize_queue, - executed_test_files: all_test_file_paths + executed_test_files: node_test_file_paths ) if test_file_paths.empty? - unless all_test_file_paths.empty? + unless node_test_file_paths.empty? KnapsackPro::Adapters::MinitestAdapter.verify_bind_method_called end @@ -76,7 +76,7 @@ def self.run_tests(accumulator) KnapsackPro::Hooks::Queue.call_before_subset_queue - all_test_file_paths += test_file_paths + node_test_file_paths += test_file_paths result = minitest_run(runner, test_file_paths, args) exitstatus = 1 unless result @@ -91,7 +91,7 @@ def self.run_tests(accumulator) can_initialize_queue: false, args: args, exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, + node_test_file_paths: node_test_file_paths, } end end diff --git a/lib/knapsack_pro/runners/queue/rspec_runner.rb b/lib/knapsack_pro/runners/queue/rspec_runner.rb index 26836400..c0c8138e 100644 --- a/lib/knapsack_pro/runners/queue/rspec_runner.rb +++ b/lib/knapsack_pro/runners/queue/rspec_runner.rb @@ -4,223 +4,174 @@ module KnapsackPro module Runners module Queue class RSpecRunner < BaseRunner - @@used_seed = nil - - def self.run(args) + def self.run(args, stream_error = $stderr, stream_out = $stdout) require 'rspec/core' + require_relative '../../extensions/rspec_extension' require_relative '../../formatters/time_tracker' require_relative '../../formatters/time_tracker_fetcher' - require_relative '../../formatters/rspec_queue_summary_formatter' - require_relative '../../formatters/rspec_queue_profile_formatter_extension' + + KnapsackPro::Extensions::RSpecExtension.setup! ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_rspec - ENV['KNAPSACK_PRO_QUEUE_RECORDING_ENABLED'] = 'true' - ENV['KNAPSACK_PRO_QUEUE_ID'] = KnapsackPro::Config::EnvGenerator.set_queue_id - KnapsackPro::Config::Env.set_test_runner_adapter(adapter_class) - runner = new(adapter_class) - - cli_args = (args || '').split - adapter_class.ensure_no_tag_option_when_rspec_split_by_test_examples_enabled!(cli_args) - - # when format option is not defined by user then use progress formatter to show tests execution progress - cli_args += ['--format', 'progress'] unless adapter_class.has_format_option?(cli_args) - - cli_args += [ - # shows summary of all tests executed in Queue Mode at the very end - '--format', KnapsackPro::Formatters::RSpecQueueSummaryFormatter.to_s, - '--format', KnapsackPro::Formatters::TimeTracker.to_s, - '--default-path', runner.test_dir, - ] - - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: cli_args, - exitstatus: 0, - all_test_file_paths: [], - } - while accumulator[:status] == :next - handle_signal! - accumulator = run_tests(accumulator) - end + rspec_pure = KnapsackPro::Pure::Queue::RSpecPure.new - Kernel.exit(accumulator[:exitstatus]) + queue_runner = new(KnapsackPro::Adapters::RSpecAdapter, rspec_pure, args, stream_error, stream_out) + queue_runner.run end - def self.run_tests(accumulator) - runner = accumulator.fetch(:runner) - can_initialize_queue = accumulator.fetch(:can_initialize_queue) - args = accumulator.fetch(:args) - exitstatus = accumulator.fetch(:exitstatus) - all_test_file_paths = accumulator.fetch(:all_test_file_paths) + def initialize(adapter_class, rspec_pure, args, stream_error, stream_out) + super(adapter_class) + @adapter_class = adapter_class + @rspec_pure = rspec_pure + has_format_option = @adapter_class.has_format_option?((args || '').split) + @cli_args = rspec_pure.prepare_cli_args(args, has_format_option, test_dir) + @stream_error = stream_error + @stream_out = stream_out + @node_test_file_paths = [] + @rspec_runner = nil # RSpec::Core::Runner is lazy initialized + end - test_file_paths = runner.test_file_paths( - can_initialize_queue: can_initialize_queue, - executed_test_files: all_test_file_paths - ) + # Based on: + # https://github.com/rspec/rspec-core/blob/f8c8880dabd8f0544a6f91d8d4c857c1bd8df903/lib/rspec/core/runner.rb#L85 + # + # @return [Fixnum] exit status code. + # 0 if all specs passed, + # or the configured failure exit code (1 by default) if specs failed. + def run + pre_run_setup + + if @rspec_runner.knapsack__wants_to_quit? + exit_code = @rspec_runner.knapsack__exit_early + Kernel.exit(exit_code) + end + + begin + exit_code = @rspec_runner.knapsack__run_specs(self) + rescue KnapsackPro::Runners::Queue::BaseRunner::TerminationError + exit_code = @rspec_pure.error_exit_code(@rspec_runner.knapsack__error_exit_code) + Kernel.exit(exit_code) + rescue Exception => exception + KnapsackPro.logger.error("An unexpected exception happened. RSpec cannot handle it. The exception: #{exception.inspect}") - if test_file_paths.empty? - unless all_test_file_paths.empty? - KnapsackPro::Adapters::RSpecAdapter.verify_bind_method_called + message = @rspec_pure.exit_summary(unexecuted_test_files) + KnapsackPro.logger.warn(message) if message - KnapsackPro::Formatters::RSpecQueueSummaryFormatter.print_summary - KnapsackPro::Formatters::RSpecQueueProfileFormatterExtension.print_summary + exit_code = @rspec_pure.error_exit_code(@rspec_runner.knapsack__error_exit_code) + Kernel.exit(exit_code) + end - args += ['--seed', @@used_seed] if @@used_seed + post_run_tasks(exit_code) + end - log_rspec_command(args, all_test_file_paths, :end_of_queue) - end + def with_batch + can_initialize_queue = true - KnapsackPro::Hooks::Queue.call_after_queue + loop do + handle_signal! + test_file_paths = pull_tests_from_queue(can_initialize_queue: can_initialize_queue) + can_initialize_queue = false - time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call - KnapsackPro::Report.save_node_queue_to_api(time_tracker&.queue(all_test_file_paths) || []) + break if test_file_paths.empty? - return { - status: :completed, - exitstatus: exitstatus, - } - else subset_queue_id = KnapsackPro::Config::EnvGenerator.set_subset_queue_id ENV['KNAPSACK_PRO_SUBSET_QUEUE_ID'] = subset_queue_id KnapsackPro::Hooks::Queue.call_before_subset_queue - all_test_file_paths += test_file_paths - cli_args = args + test_file_paths - - ensure_spec_opts_have_knapsack_pro_formatters - options = ::RSpec::Core::ConfigurationOptions.new(cli_args) - rspec_runner = ::RSpec::Core::Runner.new(options) - - begin - exit_code = rspec_runner.run($stderr, $stdout) - exitstatus = exit_code if exit_code != 0 - rescue Exception => exception - KnapsackPro.logger.error("Having exception when running RSpec: #{exception.inspect}") - KnapsackPro::Formatters::RSpecQueueSummaryFormatter.print_exit_summary(all_test_file_paths) - KnapsackPro::Hooks::Queue.call_after_subset_queue - KnapsackPro::Hooks::Queue.call_after_queue - Kernel.exit(1) - raise - else - if rspec_runner.world.wants_to_quit - KnapsackPro.logger.warn('RSpec wants to quit.') - set_terminate_process - end - if rspec_runner.world.respond_to?(:rspec_is_quitting) && rspec_runner.world.rspec_is_quitting - KnapsackPro.logger.warn('RSpec is quitting.') - set_terminate_process - end - - printable_args = args_with_seed_option_added_when_viable(args, rspec_runner) - log_rspec_command(printable_args, test_file_paths, :subset_queue) - - rspec_clear_examples - - KnapsackPro::Hooks::Queue.call_after_subset_queue - - return { - status: :next, - runner: runner, - can_initialize_queue: false, - args: args, - exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, - } - end - end - end + yield test_file_paths - def self.ensure_spec_opts_have_knapsack_pro_formatters - return unless ENV['SPEC_OPTS'] + KnapsackPro::Hooks::Queue.call_after_subset_queue - if [ - ENV['SPEC_OPTS'].include?(KnapsackPro::Formatters::RSpecQueueSummaryFormatter.to_s), - ENV['SPEC_OPTS'].include?(KnapsackPro::Formatters::TimeTracker.to_s), - ].all? - return - end + if @rspec_runner.knapsack__wants_to_quit? + KnapsackPro.logger.warn('RSpec wants to quit.') + set_terminate_process + end + if @rspec_runner.knapsack__rspec_is_quitting? + KnapsackPro.logger.warn('RSpec is quitting.') + set_terminate_process + end - unless ENV['SPEC_OPTS'].include?(KnapsackPro::Formatters::RSpecQueueSummaryFormatter.to_s) - ENV['SPEC_OPTS'] = "#{ENV['SPEC_OPTS']} --format #{KnapsackPro::Formatters::RSpecQueueSummaryFormatter}" + log_rspec_batch_command(test_file_paths) end + end - unless ENV['SPEC_OPTS'].include?(KnapsackPro::Formatters::TimeTracker.to_s) - ENV['SPEC_OPTS'] = "#{ENV['SPEC_OPTS']} --format #{KnapsackPro::Formatters::TimeTracker}" - end + def handle_signal! + self.class.handle_signal! + end + + def log_fail_fast_limit_met + KnapsackPro.logger.warn('Test execution has been canceled because the RSpec --fail-fast option is enabled. It will cause other CI nodes to run tests longer because they need to consume more tests from the Knapsack Pro Queue API.') end private - def self.adapter_class - KnapsackPro::Adapters::RSpecAdapter + def pre_run_setup + ENV['KNAPSACK_PRO_QUEUE_RECORDING_ENABLED'] = 'true' + ENV['KNAPSACK_PRO_QUEUE_ID'] = KnapsackPro::Config::EnvGenerator.set_queue_id + + KnapsackPro::Config::Env.set_test_runner_adapter(@adapter_class) + + ENV['SPEC_OPTS'] = @rspec_pure.add_knapsack_pro_formatters_to(ENV['SPEC_OPTS']) + @adapter_class.ensure_no_tag_option_when_rspec_split_by_test_examples_enabled!(@cli_args) + + rspec_configuration_options = ::RSpec::Core::ConfigurationOptions.new(@cli_args) + @rspec_runner = ::RSpec::Core::Runner.new(rspec_configuration_options) + @rspec_runner.knapsack__setup(@stream_error, @stream_out) + + ensure_no_deprecated_run_all_when_everything_filtered_option! end - def self.log_rspec_command(cli_args, test_file_paths, type) - case type - when :subset_queue - KnapsackPro.logger.info("To retry the last batch of tests fetched from the API Queue, please run the following command on your machine:") - when :end_of_queue - KnapsackPro.logger.info("To retry all the tests assigned to this CI node, please run the following command on your machine:") - end + def post_run_tasks(exit_code) + @adapter_class.verify_bind_method_called - stringified_cli_args = cli_args.join(' ') - .sub(" --format #{KnapsackPro::Formatters::RSpecQueueSummaryFormatter}", '') - .sub(" --format #{KnapsackPro::Formatters::TimeTracker}", '') + log_rspec_queue_command - KnapsackPro.logger.info( - "bundle exec rspec #{stringified_cli_args} " + - KnapsackPro::TestFilePresenter.stringify_paths(test_file_paths) - ) + time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call + KnapsackPro::Report.save_node_queue_to_api(time_tracker&.queue(@node_test_file_paths)) + + Kernel.exit(exit_code) end - # Clear rspec examples without the shared examples: - # https://github.com/rspec/rspec-core/pull/2379 - # - # Keep formatters and report to accumulate info about failed/pending tests - def self.rspec_clear_examples - if ::RSpec::ExampleGroups.respond_to?(:remove_all_constants) - ::RSpec::ExampleGroups.remove_all_constants - else - ::RSpec::ExampleGroups.constants.each do |constant| - ::RSpec::ExampleGroups.__send__(:remove_const, constant) - end - end - ::RSpec.world.example_groups.clear - ::RSpec.configuration.start_time = ::RSpec::Core::Time.now - - if KnapsackPro::Config::Env.rspec_split_by_test_examples? - # Reset example group counts to ensure scoped example ids in metadata - # have correct index (not increased by each subsequent run). - # Solves this problem: https://github.com/rspec/rspec-core/issues/2721 - ::RSpec.world.instance_variable_set(:@example_group_counts_by_spec_file, Hash.new(0)) - end + def ensure_no_deprecated_run_all_when_everything_filtered_option! + return unless @rspec_runner.knapsack__deprecated_run_all_when_everything_filtered_enabled? - # skip reset filters for old RSpec versions - if ::RSpec.configuration.respond_to?(:reset_filters) - ::RSpec.configuration.reset_filters - end + error_message = "The run_all_when_everything_filtered option is deprecated. See: #{KnapsackPro::Urls::RSPEC__DEPRECATED_RUN_ALL_WHEN_EVERYTHING_FILTERED}" + KnapsackPro.logger.error(error_message) + raise error_message end - def self.args_with_seed_option_added_when_viable(args, rspec_runner) - order_option = adapter_class.order_option(args) + def pull_tests_from_queue(can_initialize_queue: false) + test_file_paths = test_file_paths( + can_initialize_queue: can_initialize_queue, + executed_test_files: @node_test_file_paths + ) + @node_test_file_paths += test_file_paths + test_file_paths + end - if order_option - # Don't add the seed option for order other than random, e.g. `defined` - return args unless order_option.include?('rand') - # Don't add the seed option if the seed is already set in args, e.g. `rand:12345` - return args if order_option.to_s.split(':')[1] - end + def log_rspec_batch_command(test_file_paths) + order_option = @adapter_class.order_option(@cli_args) + printable_args = @rspec_pure.args_with_seed_option_added_when_viable(order_option, @rspec_runner.knapsack__seed, @cli_args) + messages = @rspec_pure.rspec_command(printable_args, test_file_paths, :batch_finished) + log_info_messages(messages) + end - # Don't add the seed option if the seed was not used (i.e. a different order is being used, e.g. `defined`) - return args unless rspec_runner.configuration.seed_used? + def log_rspec_queue_command + order_option = @adapter_class.order_option(@cli_args) + printable_args = @rspec_pure.args_with_seed_option_added_when_viable(order_option, @rspec_runner.knapsack__seed, @cli_args) + messages = @rspec_pure.rspec_command(printable_args, @node_test_file_paths, :queue_finished) + log_info_messages(messages) + end - @@used_seed = rspec_runner.configuration.seed.to_s + def log_info_messages(messages) + messages.each do |message| + KnapsackPro.logger.info(message) + end + end - args + ['--seed', @@used_seed] + def unexecuted_test_files + KnapsackPro::Formatters::TimeTrackerFetcher.unexecuted_test_files(@node_test_file_paths) end end end diff --git a/lib/knapsack_pro/urls.rb b/lib/knapsack_pro/urls.rb index d323e089..76b9aa23 100644 --- a/lib/knapsack_pro/urls.rb +++ b/lib/knapsack_pro/urls.rb @@ -28,6 +28,8 @@ module Urls REGULAR_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_TRUE_AND_POSITIVE_RETRY_COUNT = "#{HOST}/perma/ruby/regular-mode-connection-error-with-fallback-enabled-true-and-positive-retry-count" + RSPEC__DEPRECATED_RUN_ALL_WHEN_EVERYTHING_FILTERED = "#{HOST}/perma/ruby/rspec-deprecated-run-all-when-everything-filtered" + RSPEC__SKIPS_TESTS = "#{HOST}/perma/ruby/rspec-skips-tests" RSPEC__SPLIT_BY_TEST_EXAMPLES__TAG = "#{HOST}/perma/ruby/rspec-split-by-test-examples-tag" diff --git a/spec/integration/runners/queue/rspec_runner.rb b/spec/integration/runners/queue/rspec_runner.rb new file mode 100644 index 00000000..ee81fb00 --- /dev/null +++ b/spec/integration/runners/queue/rspec_runner.rb @@ -0,0 +1,80 @@ +require 'knapsack_pro' +require 'json' + +ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC'] = SecureRandom.hex +ENV['KNAPSACK_PRO_CI_NODE_BUILD_ID'] = SecureRandom.uuid +ENV['KNAPSACK_PRO_TEST_DIR'] = 'spec_integration' +ENV['KNAPSACK_PRO_TEST_FILE_PATTERN'] = "spec_integration/**{,/*/**}/*_spec.rb" + +RSPEC_OPTIONS = ENV.fetch('TEST__RSPEC_OPTIONS') +SHOW_DEBUG_LOG = ENV['TEST__SHOW_DEBUG_LOG'] == 'true' +SPEC_BATCHES = JSON.load(ENV.fetch('TEST__SPEC_BATCHES')) + +class IntegrationTestLogger + def self.log(message) + puts "[INTEGRATION TEST] #{message}" + end +end + +module KnapsackProExtensions + module QueueAllocatorExtension + def test_file_paths(can_initialize_queue, executed_test_files) + @batch_index ||= 0 + last_batch = [] + batches = [*SPEC_BATCHES, last_batch] + tests = batches[@batch_index] + @batch_index += 1 + + if SHOW_DEBUG_LOG + IntegrationTestLogger.log("Mocked tests from the Queue API: #{tests.inspect}") + end + + tests + end + end + + module Report + def create_build_subset(test_files) + if ENV['TEST__LOG_EXECUTION_TIMES'] + have_execution_time = test_files.all? { _1.fetch('time_execution') > 0 } + IntegrationTestLogger.log("test_files: #{test_files.size}, test files have execution time: #{have_execution_time}") + end + + return unless SHOW_DEBUG_LOG + IntegrationTestLogger.log("Mocked the #{__method__} method") + end + end + + module RSpecAdapter + def test_file_cases_for(slow_test_files) + IntegrationTestLogger.log("Mocked test file cases for slow test files: #{slow_test_files}") + + test_file_paths = JSON.load(ENV.fetch('TEST__TEST_FILE_CASES_FOR_SLOW_TEST_FILES')) + test_file_paths.map do |path| + { 'path' => path } + end + end + end +end + +KnapsackPro::QueueAllocator.prepend(KnapsackProExtensions::QueueAllocatorExtension) + +module KnapsackPro + class Report + class << self + prepend KnapsackProExtensions::Report + end + end +end + +module KnapsackPro + module Adapters + class RSpecAdapter + class << self + prepend KnapsackProExtensions::RSpecAdapter + end + end + end +end + +KnapsackPro::Runners::Queue::RSpecRunner.run(RSPEC_OPTIONS) diff --git a/spec/integration/runners/queue/rspec_runner_spec.rb b/spec/integration/runners/queue/rspec_runner_spec.rb new file mode 100644 index 00000000..ec203ff4 --- /dev/null +++ b/spec/integration/runners/queue/rspec_runner_spec.rb @@ -0,0 +1,2232 @@ +require 'open3' +require 'json' +require 'nokogiri' + +describe "#{KnapsackPro::Runners::Queue::RSpecRunner} - Integration tests", :clear_tmp do + SPEC_DIRECTORY = 'spec_integration' + + class Spec + attr_reader :path, :content + + def initialize(path, content) + @path = "#{SPEC_DIRECTORY}/#{path}" + @content = content + end + end + + # @param rspec_options String + # @param spec_batches Array[Array[String]] + def generate_specs(spec_helper, rspec_options, spec_batches) + ENV['TEST__RSPEC_OPTIONS'] = rspec_options + + spec_helper_path = "#{SPEC_DIRECTORY}/spec_helper.rb" + File.open(spec_helper_path, 'w') { |file| file.write(spec_helper) } + + paths = spec_batches.flatten.map do |spec_item| + File.open(spec_item.path, 'w') { |file| file.write(spec_item.content) } + spec_item.path + end + + stub_spec_batches( + spec_batches.map { _1.map(&:path) } + ) + end + + def stub_spec_batches(batched_tests) + ENV['TEST__SPEC_BATCHES'] = batched_tests.to_json + end + + # @param test_file_paths Array[String] + # Example: ['spec_integration/a_spec.rb[1:1]'] + def mock_test_cases_for_slow_test_files(test_file_paths) + ENV['TEST__TEST_FILE_CASES_FOR_SLOW_TEST_FILES'] = test_file_paths.to_json + end + + def log_command_result(stdout, stderr, status) + return if ENV['TEST__SHOW_DEBUG_LOG'] != 'true' + + puts '='*50 + puts 'STDOUT:' + puts stdout + puts + + puts '='*50 + puts 'STDERR:' + puts stderr + puts + + puts '='*50 + puts 'Exit status code:' + puts status + puts + end + + let(:spec_helper_with_knapsack) do + <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + SPEC + end + + subject do + command = 'ruby spec/integration/runners/queue/rspec_runner.rb' + stdout, stderr, status = Open3.capture3(command) + log_command_result(stdout, stderr, status) + OpenStruct.new(stdout: stdout, stderr: stderr, exit_code: status.exitstatus) + end + + before do + FileUtils.mkdir_p(SPEC_DIRECTORY) + + ENV['KNAPSACK_PRO_LOG_LEVEL'] = 'debug' + # Useful when creating or editing a test: + # ENV['TEST__SHOW_DEBUG_LOG'] = 'true' + end + after do + FileUtils.rm_rf(SPEC_DIRECTORY) + FileUtils.mkdir_p(SPEC_DIRECTORY) + + ENV.delete('KNAPSACK_PRO_LOG_LEVEL') + ENV.keys.select { _1.start_with?('TEST__') }.each do |key| + ENV.delete(key) + end + end + + context 'when a few batches of tests returned by the Queue API' do + it 'runs tests' do + rspec_options = '--format d' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('DEBUG -- : [knapsack_pro] Queue Mode enabled.') + + expect(actual.stdout).to include('A1 test example') + expect(actual.stdout).to include('B1 test example') + expect(actual.stdout).to include('C1 test example') + + expect(actual.stdout).to include('INFO -- : [knapsack_pro] To retry the last batch of tests fetched from the Queue API, please run the following command on your machine:') + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --format d --default-path spec_integration "spec_integration/a_spec.rb" "spec_integration/b_spec.rb"') + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --format d --default-path spec_integration "spec_integration/c_spec.rb"') + + expect(actual.stdout).to include('INFO -- : [knapsack_pro] To retry all the tests assigned to this CI node, please run the following command on your machine:') + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --format d --default-path spec_integration "spec_integration/a_spec.rb" "spec_integration/b_spec.rb" "spec_integration/c_spec.rb"') + + expect(actual.stdout).to include('3 examples, 0 failures') + + expect(actual.stdout).to include('DEBUG -- : [knapsack_pro] Global test execution duration:') + + expect(actual.exit_code).to eq 0 + end + + it 'detects test execution times correctly before sending it to API' do + ENV['TEST__LOG_EXECUTION_TIMES'] = 'true' + + rspec_options = '--format d' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('[INTEGRATION TEST] test_files: 3, test files have execution time: true') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when spec_helper.rb has a missing KnapsackPro::Adapters::RSpecAdapter.bind method' do + it do + rspec_options = '' + + spec_helper = <<~SPEC + require 'knapsack_pro' + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stdout).to include('ERROR -- : [knapsack_pro] You forgot to call KnapsackPro::Adapters::RSpecAdapter.bind method in your test runner configuration file. It is needed to record test files time execution. Please follow the installation guide to configure your project properly https://knapsackpro.com/perma/ruby/installation-guide') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when RSpec options are not set' do + before do + ENV['KNAPSACK_PRO_LOG_LEVEL'] = 'info' + end + + after do + ENV.delete('KNAPSACK_PRO_LOG_LEVEL') + end + + it 'uses a default progress formatter' do + rspec_options = '' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it {} + it {} + it {} + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it {} + it {} + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it {} + it {} + it {} + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + beginning_of_knapsack_pro_log_info_message = 'I, [' + + # shows dots for the 1st batch of tests + expect(actual.stdout).to include('.....' + beginning_of_knapsack_pro_log_info_message) + # shows dots for the 2nd batch of tests + expect(actual.stdout).to include('...' + beginning_of_knapsack_pro_log_info_message) + + expect(actual.exit_code).to eq 0 + end + end + + context 'when RSpec options are not set AND Knapsack Pro log level is warn' do + before do + ENV['KNAPSACK_PRO_LOG_LEVEL'] = 'warn' + ENV.delete('TEST__SHOW_DEBUG_LOG') + end + after do + ENV.delete('KNAPSACK_PRO_LOG_LEVEL') + end + + it 'uses a default progress formatter AND shows dots for all test examples' do + rspec_options = '' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it {} + it {} + it {} + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it {} + it {} + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it {} + it {} + it {} + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('.'*8) + + expect(actual.exit_code).to eq 0 + end + end + + context 'when hooks are defined' do + it 'calls RSpec before/after hooks only once for multiple batches of tests' do + rspec_options = '' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.before(:suite) do + puts 'RSpec_before_suite_hook' + end + config.after(:suite) do + puts 'RSpec_after_suite_hook' + end + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout.scan(/RSpec_before_suite_hook/).size).to eq 1 + expect(actual.stdout.scan(/RSpec_after_suite_hook/).size).to eq 1 + + expect(actual.exit_code).to eq 0 + end + + it 'calls queue hooks for multiple batches of tests (queue hooks can be defined multiple times)' do + rspec_options = '' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + KnapsackPro::Hooks::Queue.before_queue do |queue_id| + puts '1st before_queue - run before the test suite' + end + KnapsackPro::Hooks::Queue.before_queue do |queue_id| + puts '2nd before_queue - run before the test suite' + end + + KnapsackPro::Hooks::Queue.before_subset_queue do |queue_id, subset_queue_id| + puts '1st before_subset_queue - run before the next subset of tests' + end + KnapsackPro::Hooks::Queue.before_subset_queue do |queue_id, subset_queue_id| + puts '2nd before_subset_queue - run before the next subset of tests' + end + + KnapsackPro::Hooks::Queue.after_subset_queue do |queue_id, subset_queue_id| + puts '1st after_subset_queue - run after the previous subset of tests' + end + KnapsackPro::Hooks::Queue.after_subset_queue do |queue_id, subset_queue_id| + puts '2nd after_subset_queue - run after the previous subset of tests' + end + + KnapsackPro::Hooks::Queue.after_queue do |queue_id| + puts '1st after_queue - run after the test suite' + end + KnapsackPro::Hooks::Queue.after_queue do |queue_id| + puts '2nd after_queue - run after the test suite' + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout.scan(/1st before_queue - run before the test suite/).size).to eq 1 + expect(actual.stdout.scan(/2nd before_queue - run before the test suite/).size).to eq 1 + expect(actual.stdout.scan(/1st before_subset_queue - run before the next subset of tests/).size).to eq 2 + expect(actual.stdout.scan(/2nd before_subset_queue - run before the next subset of tests/).size).to eq 2 + expect(actual.stdout.scan(/1st after_subset_queue - run after the previous subset of tests/).size).to eq 2 + expect(actual.stdout.scan(/2nd after_subset_queue - run after the previous subset of tests/).size).to eq 2 + expect(actual.stdout.scan(/1st after_queue - run after the test suite/).size).to eq 1 + expect(actual.stdout.scan(/2nd after_queue - run after the test suite/).size).to eq 1 + + expect(actual.exit_code).to eq 0 + end + + it 'calls hooks defined with when_first_matching_example_defined only once for multiple batches of tests' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + def when_first_matching_example_defined(type:) + env_var_name = "WHEN_FIRST_MATCHING_EXAMPLE_DEFINED_FOR_" + type.to_s.upcase + + RSpec.configure do |config| + config.when_first_matching_example_defined(type: type) do + config.before(:context) do + unless ENV[env_var_name] + yield + end + ENV[env_var_name] = 'hook_called' + end + end + end + end + + when_first_matching_example_defined(type: :model) do + puts 'RSpec_custom_hook_called_once_for_model' + end + + when_first_matching_example_defined(type: :system) do + puts 'RSpec_custom_hook_called_once_for_system' + end + + RSpec.configure do |config| + config.before(:suite) do + puts 'RSpec_before_suite_hook' + end + + config.when_first_matching_example_defined(type: :model) do + config.before(:suite) do + puts 'RSpec_before_suite_hook_for_model' + end + end + + config.when_first_matching_example_defined(type: :system) do + config.before(:suite) do + puts 'RSpec_before_suite_hook_for_system' + end + end + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe', type: :model do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe', type: :system do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + + it 'C1 test example', :model do + expect(1).to eq 1 + end + end + SPEC + + spec_d = Spec.new('d_spec.rb', <<~SPEC) + describe 'D_describe', type: :system do + it 'D1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + [spec_c], + [spec_d], + ]) + + actual = subject + + expect(actual.stdout.scan(/RSpec_before_suite_hook/).size).to eq 1 + + # skips before(:suite) hooks that were defined too late in 1st & 2nd batch of tests after before(:suite) hook is already executed + expect(actual.stdout.scan(/RSpec_before_suite_hook_for_model/).size).to eq 0 + expect(actual.stdout.scan(/RSpec_before_suite_hook_for_system/).size).to eq 0 + + expect(actual.stdout.scan(/RSpec_custom_hook_called_once_for_model/).size).to eq 1 + expect(actual.stdout.scan(/RSpec_custom_hook_called_once_for_system/).size).to eq 1 + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the RSpec seed is used' do + it do + rspec_options = '--order rand:123' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('Randomized with seed 123') + + # 1st batch + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --order rand:123 --format progress --default-path spec_integration "spec_integration/a_spec.rb" "spec_integration/b_spec.rb"') + # 2nd batch + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --order rand:123 --format progress --default-path spec_integration "spec_integration/c_spec.rb"') + + # the final RSpec command with seed + expect(actual.stdout).to include('INFO -- : [knapsack_pro] bundle exec rspec --order rand:123 --format progress --default-path spec_integration "spec_integration/a_spec.rb" "spec_integration/b_spec.rb" "spec_integration/c_spec.rb"') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when a failing test in a batch of tests that is not the last batch fetched from the Queue API' do + it 'returns 1 as exit code (it remembers that one of the batches has a failing test)' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + failing_spec = Spec.new('failing_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, failing_spec], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('B1 test example (FAILED - 1)') + expect(actual.stdout).to include('Failure/Error: expect(1).to eq 0') + expect(actual.stdout).to include('3 examples, 1 failure') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when a failing test raises an exception' do + it 'returns 1 as exit code AND the exception does not leak outside of the RSpec runner context' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + failing_spec = Spec.new('failing_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + raise 'A custom exception from a test' + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, failing_spec], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('B1 test example (FAILED - 1)') + expect(actual.stdout).to include("Failure/Error: raise 'A custom exception from a test'") + expect(actual.stdout).to include('3 examples, 1 failure') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when a spec file has a syntax error outside of the test example' do + it 'stops running tests on the batch that has a test file with the syntax error AND returns 1 as exit code' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + failing_spec = Spec.new('failing_spec.rb', <<~SPEC) + describe 'B_describe' do + a_fake_method + + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [failing_spec], + [spec_c], + ]) + + actual = subject + + # 1st batch of tests executed correctly + expect(actual.stdout).to include('A1 test example') + # 2nd batch contains the test file that cannot be loaded and the test file is not executed + expect(actual.stdout).to_not include('B1 test example') + # 3rd batch is never executed + expect(actual.stdout).to_not include('C1 test example') + + expect(actual.stdout).to include('An error occurred while loading ./spec_integration/failing_spec.rb') + expect(actual.stdout).to match(/undefined local variable or method `a_fake_method' for.* RSpec::ExampleGroups::BDescribe/) + expect(actual.stdout).to include('WARN -- : [knapsack_pro] RSpec wants to quit') + expect(actual.stdout).to include('1 example, 0 failures, 1 error occurred outside of examples') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when a syntax error (an exception) in spec_helper.rb' do + it 'exits early with 1 as the exit code without running tests because RSpec wants to quit' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + a_fake_method + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stdout).to include('An error occurred while loading spec_helper.') + expect(actual.stdout).to include("undefined local variable or method `a_fake_method' for main") + expect(actual.stdout).to include('0 examples, 0 failures, 1 error occurred outside of examples') + + expect(actual.exit_code).to eq 1 + end + end + + # Based on: + # https://github.com/rspec/rspec-core/pull/2926/files + context 'when RSpec is quitting' do + let(:helper_with_exit_location) { "#{SPEC_DIRECTORY}/helper_with_exit.rb" } + + it 'returns non zero exit code because RSpec is quitting' do + skip 'Not supported by this RSpec version' if RSpec::Core::Version::STRING == '3.10.2' + + File.open(helper_with_exit_location, 'w') { |file| file.write('exit 123') } + + rspec_options = "--format documentation --require ./#{helper_with_exit_location}" + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stdout).to include('While loading ./spec_integration/helper_with_exit.rb an `exit` / `raise SystemExit` occurred, RSpec will now quit.') + + expect(actual.stdout).to_not include('A1 test example') + expect(actual.stdout).to_not include('B1 test example') + + expect(actual.exit_code).to eq 123 + end + end + + context 'when the test suite has pending tests' do + it 'shows the summary of pending tests' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + xit 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + + xit 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('B1 test example (PENDING: Temporarily skipped with xit)') + expect(actual.stdout).to include('C2 test example (PENDING: Temporarily skipped with xit)') + + expect(actual.stdout).to include("Pending: (Failures listed here are expected and do not affect your suite's status)") + expect(actual.stdout).to include('1) B_describe B1 test example') + expect(actual.stdout).to include('2) C_describe C2 test example') + + expect(actual.stdout).to include('4 examples, 0 failures, 2 pending') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when a test file raises an exception that cannot be handle by RSpec' do + it 'stops running tests when unhandled exception happens AND sets 1 as exit code AND shows summary of unexecuted tests' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + # list of unhandled exceptions: + # RSpec::Support::AllExceptionsExceptOnesWeMustNotRescue::AVOID_RESCUING + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + raise NoMemoryError.new + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('A1 test example') + + expect(actual.stdout).to include('B_describe') + expect(actual.stdout).to include('An unexpected exception happened. RSpec cannot handle it. The exception: #') + expect(actual.stdout).to_not include('B1 test example') + + expect(actual.stdout).to_not include('C1 test example') + + # 2nd test example raised unhandled exception during runtime. + # It breaks RSpec so it was not marked as failed. + expect(actual.stdout).to include('2 examples, 0 failures') + + expect(actual.stdout).to include('WARN -- : [knapsack_pro] Unexecuted tests on this CI node (including pending tests): spec_integration/b_spec.rb') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when a test file raises an exception that cannot be handle by RSpec AND --error-exit-code is set' do + it 'sets a custom exit code' do + rspec_options = '--format documentation --error-exit-code 2' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + raise NoMemoryError.new + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.exit_code).to eq 2 + end + end + + context 'when a termination signal is received by the process' do + it 'terminates the process after tests from the current RSpec ExampleGroup are executed and sets 1 as exit code' do + rspec_options = '--format documentation' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B1_describe' do + describe 'B1.1_describe' do + xit 'B1.1.1 test example' do + expect(1).to eq 1 + end + it 'B1.1.2 test example' do + Process.kill("INT", Process.pid) + end + it 'B1.1.3 test example' do + expect(1).to eq 0 + end + end + + describe 'B1.2_describe' do + it 'B1.2.1 test example' do + expect(1).to eq 1 + end + end + end + + describe 'B2_describe' do + it 'B2.1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_d = Spec.new('d_spec.rb', <<~SPEC) + describe 'D_describe' do + it 'D1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b, spec_c], + [spec_d], + ]) + + actual = subject + + expect(actual.stdout).to include('B1.1.1 test example (PENDING: Temporarily skipped with xit)') + expect(actual.stdout).to include('INT signal has been received. Terminating Knapsack Pro...') + expect(actual.stdout).to include('B1.1.2 test example') + expect(actual.stdout).to include('B1.1.3 test example (FAILED - 1)') + expect(actual.stdout).to include('B1.2.1 test example') + + # next ExampleGroup within the same b_spec.rb is not executed + expect(actual.stdout).to_not include('B2.1 test example') + + # next test file from the same batch is not executed + expect(actual.stdout).to_not include('C1 test example') + + # next batch of tests is not pulled from the Queue API and is not executed + expect(actual.stdout).to_not include('D1 test example') + + + expect(actual.stdout).to include( + <<~OUTPUT + Pending: (Failures listed here are expected and do not affect your suite's status) + + 1) B1_describe B1.1_describe B1.1.1 test example + OUTPUT + ) + + expect(actual.stdout).to include( + <<~OUTPUT + Failures: + + 1) B1_describe B1.1_describe B1.1.3 test example + OUTPUT + ) + + expect(actual.exit_code).to eq 1 + end + end + + context 'when a termination signal is received by the process AND --error-exit-code is set' do + it 'terminates the process AND sets a custom exit code' do + rspec_options = '--format documentation --error-exit-code 3' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + Process.kill("INT", Process.pid) + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('INT signal has been received. Terminating Knapsack Pro...') + + expect(actual.exit_code).to eq 3 + end + end + + context 'when deprecated run_all_when_everything_filtered option is true' do + it 'shows an error message AND sets 1 as exit code' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.run_all_when_everything_filtered = true + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stdout).to include('ERROR -- : [knapsack_pro] The run_all_when_everything_filtered option is deprecated. See: https://knapsackpro.com/perma/ruby/rspec-deprecated-run-all-when-everything-filtered') + + expect(actual.stdout).to_not include('A1 test example') + expect(actual.stdout).to_not include('B1 test example') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when filter_run_when_matching is set to :focus and some tests are tagged with the focus tag' do + it 'shows an error message for :focus tagged tests AND sets 1 as exit code (shows the error because the batch of tests that has no focus tagged tests will run tests instead of not running them)' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.filter_run_when_matching :focus + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example', :focus do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('A1 test example') + + expect(actual.stdout).to include('B1 test example (FAILED - 1)') + expect(actual.stdout).to_not include('B2 test example') # skips B2 test due to tagged B1 + + expect(actual.stdout).to include('C1 test example') + + expect(actual.stdout).to include('Knapsack Pro found an example tagged with focus in spec_integration/b_spec.rb, please remove it. See more: https://knapsackpro.com/perma/ruby/rspec-skips-tests') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when the late CI node has an empty batch of tests because other CI nodes already consumed tests from the Queue API' do + it 'sets 0 as exit code' do + rspec_options = '--format documentation' + + generate_specs(spec_helper_with_knapsack, rspec_options, []) + + actual = subject + + expect(actual.stdout).to include('0 examples, 0 failures') + expect(actual.stdout).to include('WARN -- : [knapsack_pro] No test files were executed on this CI node.') + expect(actual.stdout).to include('DEBUG -- : [knapsack_pro] This CI node likely started work late after the test files were already executed by other CI nodes consuming the queue.') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the fail_if_no_examples option is true AND the late CI node has an empty batch of tests because other CI nodes already consumed tests from the Queue API' do + it 'sets 0 as exit code to ignore the fail_if_no_examples option' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.fail_if_no_examples = true + end + SPEC + + generate_specs(spec_helper, rspec_options, []) + + actual = subject + + expect(actual.stdout).to include('0 examples, 0 failures') + expect(actual.stdout).to include('WARN -- : [knapsack_pro] No test files were executed on this CI node.') + expect(actual.stdout).to include('DEBUG -- : [knapsack_pro] This CI node likely started work late after the test files were already executed by other CI nodes consuming the queue.') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the fail_if_no_examples option is true AND a batch of tests has a test file without test examples' do + it 'sets 0 as exit code to ignore the fail_if_no_examples option' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.fail_if_no_examples = true + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b_with_no_examples = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a], + [spec_b_with_no_examples], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('2 examples, 0 failures') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when tests are failing AND --failure-exit-code is set' do + it 'returns a custom exit code' do + rspec_options = '--format documentation --failure-exit-code 4' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + failing_spec = Spec.new('failing_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, failing_spec], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('B1 test example (FAILED - 1)') + expect(actual.stdout).to include('Failure/Error: expect(1).to eq 0') + expect(actual.stdout).to include('3 examples, 1 failure') + + expect(actual.exit_code).to eq 4 + end + end + + context 'when --profile is set' do + it 'shows top slowest examples AND top slowest example groups' do + rspec_options = '--format d --profile' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('Top 3 slowest examples') + expect(actual.stdout).to include('A_describe A1 test example') + expect(actual.stdout).to include('B_describe B1 test example') + expect(actual.stdout).to include('C_describe C1 test example') + + expect(actual.stdout).to include('Top 3 slowest example groups') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when an invalid RSpec option is set' do + it 'returns 1 as exit code AND shows an error message to stderr' do + rspec_options = '--format d --fake-rspec-option' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stderr).to include('invalid option: --fake-rspec-option') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when --fail-fast is set' do + it 'stops running tests on the failing test AND returns 1 as exit code AND shows a warning message' do + rspec_options = '--format d --fail-fast' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 0 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('A1 test example') + expect(actual.stdout).to include('B1 test example') + expect(actual.stdout).to_not include('C1 test example') + expect(actual.stdout).to_not include('C2 test example') + + expect(actual.stdout).to include('WARN -- : [knapsack_pro] Test execution has been canceled because the RSpec --fail-fast option is enabled. It will cause other CI nodes to run tests longer because they need to consume more tests from the Knapsack Pro Queue API.') + + expect(actual.stdout).to include('2 examples, 1 failure') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when the fail_fast option is set with a specific number of tests' do + it 'stops running tests on the 2nd failing test AND returns 1 as exit code AND shows a warning message when fail fast limit met' do + rspec_options = '--format d' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.fail_fast = 2 + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('A1 test example (FAILED - 1)') + expect(actual.stdout).to include('B1 test example') + expect(actual.stdout).to include('B2 test example (FAILED - 2)') + expect(actual.stdout).to_not include('C1 test example') + expect(actual.stdout).to_not include('C2 test example') + + expect(actual.stdout).to include('WARN -- : [knapsack_pro] Test execution has been canceled because the RSpec --fail-fast option is enabled. It will cause other CI nodes to run tests longer because they need to consume more tests from the Knapsack Pro Queue API.') + + expect(actual.stdout).to include('3 examples, 2 failures') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when --tag is set' do + it 'runs only tagged test examples from multiple batches of tests fetched from the Queue API' do + rspec_options = '--format d --tag my_tag' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example', :my_tag do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe', :my_tag do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_d = Spec.new('d_spec.rb', <<~SPEC) + describe 'D_describe' do + it 'D1 test example' do + expect(1).to eq 1 + end + it 'D2 test example', :my_tag do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b], + [spec_c], + [spec_d], + ]) + + actual = subject + + expect(actual.stdout).to_not include('A1 test example') + expect(actual.stdout).to include('A2 test example') + + expect(actual.stdout).to include('B1 test example') + + expect(actual.stdout).to_not include('C1 test example') + + expect(actual.stdout).to_not include('D1 test example') + expect(actual.stdout).to include('D2 test example') + + expect(actual.stdout).to include('3 examples, 0 failures') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the RSpec split by examples is enabled' do + before do + ENV['KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES'] = 'true' + + # remember to mock Queue API batches to include test examples (example: a_spec.rb[1:1]) + # for the following slow test files + ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN'] = "#{SPEC_DIRECTORY}/a_spec.rb" + end + after do + ENV.delete('KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES') + ENV.delete('KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN') + end + + it 'splits slow test files by examples AND ensures the test examples are executed only once' do + rspec_options = '--format d' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b, spec_c] + ]) + mock_test_cases_for_slow_test_files([ + "#{spec_a.path}[1:1]", + "#{spec_a.path}[1:2]", + ]) + stub_spec_batches([ + ["#{spec_a.path}[1:1]", spec_b.path], + ["#{spec_a.path}[1:2]", spec_c.path], + ]) + + actual = subject + + expect(actual.stdout).to include('DEBUG -- : [knapsack_pro] Detected 1 slow test files: [{"path"=>"spec_integration/a_spec.rb"}]') + + expect(actual.stdout).to include( + <<~OUTPUT + A_describe + A1 test example + + B_describe + B1 test example + B2 test example + OUTPUT + ) + + expect(actual.stdout).to include( + <<~OUTPUT + A_describe + A2 test example + + C_describe + C1 test example + C2 test example + OUTPUT + ) + + expect(actual.stdout.scan(/A1 test example/).size).to eq 1 + expect(actual.stdout.scan(/A2 test example/).size).to eq 1 + + expect(actual.stdout).to include('6 examples, 0 failures') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the RSpec split by examples is enabled AND --tag is set' do + before do + ENV['KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES'] = 'true' + + # remember to mock Queue API batches to include test examples (example: a_spec.rb[1:1]) + # for the following slow test files + ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN'] = "#{SPEC_DIRECTORY}/a_spec.rb" + end + after do + ENV.delete('KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES') + ENV.delete('KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN') + end + + it 'sets 1 as exit code AND raises an error (a test example path as a_spec.rb[1:1] would always be executed even when it does not have the tag that is set via the --tag option. We cannot run tests because it could lead to running unintentional tests)' do + rspec_options = '--format d --tag my_tag' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe', :my_tag do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b, spec_c] + ]) + mock_test_cases_for_slow_test_files([ + "#{spec_a.path}[1:1]", + "#{spec_a.path}[1:2]", + ]) + stub_spec_batches([ + ["#{spec_a.path}[1:1]", spec_b.path], + ["#{spec_a.path}[1:2]", spec_c.path], + ]) + + actual = subject + + expect(actual.stdout).to include('ERROR -- : [knapsack_pro] It is not allowed to use the RSpec tag option together with the RSpec split by test examples feature. Please see: https://knapsackpro.com/perma/ruby/rspec-split-by-test-examples-tag') + + expect(actual.stdout).to_not include('A1 test example') + expect(actual.stdout).to_not include('A2 test example') + expect(actual.stdout).to_not include('B1 test example') + expect(actual.stdout).to_not include('B2 test example') + expect(actual.stdout).to_not include('C1 test example') + expect(actual.stdout).to_not include('C2 test example') + + expect(actual.exit_code).to eq 1 + end + end + + context 'when the RSpec split by examples is enabled AND JSON formatter is used' do + let(:json_file) { "#{SPEC_DIRECTORY}/rspec.json" } + + before do + ENV['KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES'] = 'true' + + # remember to mock Queue API batches to include test examples (example: a_spec.rb[1:1]) + # for the following slow test files + ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN'] = "#{SPEC_DIRECTORY}/a_spec.rb" + end + after do + ENV.delete('KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES') + ENV.delete('KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN') + end + + it 'produces a JSON report' do + rspec_options = "--format documentation --format json --out ./#{json_file}" + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b, spec_c] + ]) + mock_test_cases_for_slow_test_files([ + "#{spec_a.path}[1:1]", + "#{spec_a.path}[1:2]", + ]) + stub_spec_batches([ + ["#{spec_a.path}[1:1]", spec_b.path], + ["#{spec_a.path}[1:2]", spec_c.path], + ]) + + actual = subject + + file_content = File.read(json_file) + json = JSON.load(file_content) + examples = json.fetch('examples') + + example_ids = examples.map do + _1.fetch('id') + end + expect(example_ids).to match_array([ + './spec_integration/a_spec.rb[1:1]', + './spec_integration/b_spec.rb[1:1]', + './spec_integration/b_spec.rb[1:2]', + './spec_integration/a_spec.rb[1:2]', + './spec_integration/c_spec.rb[1:1]', + './spec_integration/c_spec.rb[1:2]' + ]) + + example_full_descriptions = examples.map do + _1.fetch('full_description') + end + expect(example_full_descriptions).to match_array([ + 'A_describe A1 test example', + 'B_describe B1 test example', + 'B_describe B2 test example', + 'A_describe A2 test example', + 'C_describe C1 test example', + 'C_describe C2 test example' + ]) + + expect(json.fetch('summary_line')).to eq '6 examples, 0 failures' + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the RSpec split by examples is enabled AND JUnit XML formatter is used' do + let(:xml_file) { "#{SPEC_DIRECTORY}/rspec.xml" } + + before do + ENV['KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES'] = 'true' + + # remember to mock Queue API batches to include test examples (example: a_spec.rb[1:1]) + # for the following slow test files + ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN'] = "#{SPEC_DIRECTORY}/a_spec.rb" + end + after do + ENV.delete('KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES') + ENV.delete('KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN') + end + + it 'produces a JUnit XML report' do + rspec_options = "--format documentation --format RspecJunitFormatter --out ./#{xml_file}" + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a, spec_b, spec_c] + ]) + mock_test_cases_for_slow_test_files([ + "#{spec_a.path}[1:1]", + "#{spec_a.path}[1:2]", + ]) + stub_spec_batches([ + ["#{spec_a.path}[1:1]", spec_b.path], + ["#{spec_a.path}[1:2]", spec_c.path], + ]) + + actual = subject + + file_content = File.read(xml_file) + doc = Nokogiri::XML(file_content) + + files = doc.xpath('//testcase').map do |testcase| + testcase['file'] + end + expect(files).to eq([ + './spec_integration/a_spec.rb', + './spec_integration/b_spec.rb', + './spec_integration/b_spec.rb', + './spec_integration/a_spec.rb', + './spec_integration/c_spec.rb', + './spec_integration/c_spec.rb', + ]) + + examples = doc.xpath('//testcase').map do |testcase| + testcase['name'] + end + expect(examples).to eq([ + 'A_describe A1 test example', + 'B_describe B1 test example', + 'B_describe B2 test example', + 'A_describe A2 test example', + 'C_describe C1 test example', + 'C_describe C2 test example', + ]) + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the RSpec split by examples is enabled AND simplecov is used' do + let(:coverage_dir) { "#{KNAPSACK_PRO_TMP_DIR}/coverage" } + let(:coverage_file) { "#{coverage_dir}/index.html" } + + before do + ENV['KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES'] = 'true' + + # remember to mock Queue API batches to include test examples (example: a_spec.rb[1:1]) + # for the following slow test files + ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN'] = "#{SPEC_DIRECTORY}/a_spec.rb" + end + after do + ENV.delete('KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES') + ENV.delete('KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN') + end + + it 'produces a code coverage report' do + rspec_options = '--format documentation' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + require 'simplecov' + SimpleCov.start do + coverage_dir '#{coverage_dir}' + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + it 'A2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + it 'B2 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a, spec_b, spec_c] + ]) + mock_test_cases_for_slow_test_files([ + "#{spec_a.path}[1:1]", + "#{spec_a.path}[1:2]", + ]) + stub_spec_batches([ + ["#{spec_a.path}[1:1]", spec_b.path], + ["#{spec_a.path}[1:2]", spec_c.path], + ]) + + actual = subject + + file_content = File.read(coverage_file) + + expect(file_content).to include(spec_a.path) + expect(file_content).to include(spec_b.path) + expect(file_content).to include(spec_c.path) + + expect(actual.exit_code).to eq 0 + end + end + + context 'when the example_status_persistence_file_path option is used and multiple batches of tests are fetched from the Queue API and some tests are pending and failing' do + let(:examples_file_path) { "#{SPEC_DIRECTORY}/examples.txt" } + + after do + File.delete(examples_file_path) if File.exist?(examples_file_path) + end + + it 'runs tests AND creates the example status persistence file' do + rspec_options = '--format d' + + spec_helper = <<~SPEC + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind + + RSpec.configure do |config| + config.example_status_persistence_file_path = '#{examples_file_path}' + end + SPEC + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + xit 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 0 + end + end + SPEC + + spec_c = Spec.new('c_spec.rb', <<~SPEC) + describe 'C_describe' do + it 'C1 test example' do + expect(1).to eq 1 + end + it 'C2 test example' do + expect(1).to eq 0 + end + end + SPEC + + generate_specs(spec_helper, rspec_options, [ + [spec_a, spec_b], + [spec_c], + ]) + + actual = subject + + expect(actual.stdout).to include('4 examples, 2 failures, 1 pending') + + expect(File.exist?(examples_file_path)).to be true + + examples_file_content = File.read(examples_file_path) + + expect(examples_file_content).to include './spec_integration/a_spec.rb[1:1] | pending' + expect(examples_file_content).to include './spec_integration/b_spec.rb[1:1] | failed' + expect(examples_file_content).to include './spec_integration/c_spec.rb[1:1] | passed' + expect(examples_file_content).to include './spec_integration/c_spec.rb[1:2] | failed' + + expect(actual.exit_code).to eq 1 + end + end + + context 'when the .rspec file has RSpec options' do + let(:dot_rspec_file) { "#{SPEC_DIRECTORY}/.rspec" } + + it 'ignores options from the .rspec file' do + File.open(dot_rspec_file, 'w') { |file| file.write('--format documentation') } + + rspec_options = '' + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + ]) + + actual = subject + + expect(actual.stdout).not_to include('A1 test example') + + expect(actual.stdout).to include('1 example, 0 failures') + + expect(actual.exit_code).to eq 0 + end + end + + context 'when --options is set' do + let(:rspec_custom_options_file) { "#{SPEC_DIRECTORY}/.rspec_custom_options" } + + it 'uses options from the custom rspec file' do + rspec_custom_options = <<~FILE + --require spec_helper + --profile + FILE + File.open(rspec_custom_options_file, 'w') { |file| file.write(rspec_custom_options) } + + rspec_options = "--options ./#{rspec_custom_options_file}" + + spec_a = Spec.new('a_spec.rb', <<~SPEC) + describe 'A_describe' do + it 'A1 test example' do + expect(1).to eq 1 + end + end + SPEC + + spec_b = Spec.new('b_spec.rb', <<~SPEC) + describe 'B_describe' do + it 'B1 test example' do + expect(1).to eq 1 + end + end + SPEC + + generate_specs(spec_helper_with_knapsack, rspec_options, [ + [spec_a], + [spec_b], + ]) + + actual = subject + + expect(actual.stdout).to include('2 examples, 0 failures') + + expect(actual.stdout).to include('Top 2 slowest example groups') + + expect(actual.exit_code).to eq 0 + end + end +end diff --git a/spec/knapsack_pro/adapters/base_adapter_spec.rb b/spec/knapsack_pro/adapters/base_adapter_spec.rb index 306c5930..256e9628 100644 --- a/spec/knapsack_pro/adapters/base_adapter_spec.rb +++ b/spec/knapsack_pro/adapters/base_adapter_spec.rb @@ -159,7 +159,7 @@ it do logger = instance_double(Logger) expect(KnapsackPro).to receive(:logger).and_return(logger) - expect(logger).to receive(:debug).with('Test suite time execution recording enabled.') + expect(logger).to receive(:debug).with('Regular Mode enabled.') end it { expect(subject).to receive(:bind_time_tracker) } it { expect(subject).to receive(:bind_save_report) } @@ -168,24 +168,22 @@ context 'when queue recording enabled' do let(:queue_recording_enabled?) { true } - before do - allow(subject).to receive(:bind_before_queue_hook) - allow(subject).to receive(:bind_time_tracker) - end - - it do + it 'calls queue hooks in proper order before binding time tracker' do logger = instance_double(Logger) expect(KnapsackPro).to receive(:logger).and_return(logger) - expect(logger).to receive(:debug).with('Test suite time execution queue recording enabled.') + expect(logger).to receive(:debug).with('Queue Mode enabled.') + + expect(subject).to receive(:bind_before_queue_hook).ordered + expect(subject).to receive(:bind_after_queue_hook).ordered + expect(subject).to receive(:bind_time_tracker).ordered end - it { expect(subject).to receive(:bind_before_queue_hook) } - it { expect(subject).to receive(:bind_time_tracker) } end context 'when recording disabled' do - it { expect(subject).not_to receive(:bind_time_tracker) } it { expect(subject).not_to receive(:bind_save_report) } it { expect(subject).not_to receive(:bind_before_queue_hook) } + it { expect(subject).not_to receive(:bind_after_queue_hook) } + it { expect(subject).not_to receive(:bind_time_tracker) } end end @@ -212,4 +210,12 @@ }.to raise_error(NotImplementedError) end end + + describe '#bind_after_queue_hook' do + it do + expect { + subject.bind_after_queue_hook + }.to raise_error(NotImplementedError) + end + end end diff --git a/spec/knapsack_pro/adapters/cucumber_adapter_spec.rb b/spec/knapsack_pro/adapters/cucumber_adapter_spec.rb index 4176418b..4b3103e0 100644 --- a/spec/knapsack_pro/adapters/cucumber_adapter_spec.rb +++ b/spec/knapsack_pro/adapters/cucumber_adapter_spec.rb @@ -203,16 +203,13 @@ end end - describe '#bind_queue_mode' do + describe '#bind_after_queue_hook' do it do - expect(subject).to receive(:bind_before_queue_hook) - expect(subject).to receive(:bind_time_tracker) - expect(::Kernel).to receive(:at_exit).and_yield expect(KnapsackPro::Hooks::Queue).to receive(:call_after_subset_queue) expect(KnapsackPro::Report).to receive(:save_subset_queue_to_file) - subject.bind_queue_mode + subject.bind_after_queue_hook end end end diff --git a/spec/knapsack_pro/adapters/rspec_adapter_spec.rb b/spec/knapsack_pro/adapters/rspec_adapter_spec.rb index 3646176f..a33e6d8b 100644 --- a/spec/knapsack_pro/adapters/rspec_adapter_spec.rb +++ b/spec/knapsack_pro/adapters/rspec_adapter_spec.rb @@ -341,24 +341,13 @@ end context 'with no focus' do - let(:logger) { instance_double(Logger) } - let(:duration) { 65 } - let(:global_time) { 'Global time execution for tests: 01m 05s' } - let(:time_tracker) { instance_double(KnapsackPro::Formatters::TimeTracker) } - it 'records time for current test path' do expect(config).to receive(:around).with(:each).and_yield(current_example) - expect(config).to receive(:after).with(:suite).and_yield - expect(::RSpec).to receive(:configure).twice.and_yield(config) + expect(config).to receive(:append_after).with(:suite) + expect(::RSpec).to receive(:configure).at_least(1).and_yield(config) expect(current_example).to receive(:run) - expect(time_tracker).to receive(:batch_duration).and_return(duration) - expect(KnapsackPro::Formatters::TimeTrackerFetcher).to receive(:call).and_return(time_tracker) - - expect(KnapsackPro).to receive(:logger).and_return(logger) - expect(logger).to receive(:debug).with(global_time) - subject.bind_time_tracker end end @@ -378,16 +367,5 @@ subject.bind_save_report end end - - describe '#bind_before_queue_hook' do - it do - expect(config).to receive(:before).with(:suite).and_yield - expect(::RSpec).to receive(:configure).and_yield(config) - - expect(KnapsackPro::Hooks::Queue).to receive(:call_before_queue) - - subject.bind_before_queue_hook - end - end end end diff --git a/spec/knapsack_pro/config/env_spec.rb b/spec/knapsack_pro/config/env_spec.rb index c692636e..fc7028c7 100644 --- a/spec/knapsack_pro/config/env_spec.rb +++ b/spec/knapsack_pro/config/env_spec.rb @@ -511,40 +511,6 @@ end end - describe '.modify_default_rspec_formatters' do - subject { described_class.modify_default_rspec_formatters } - - context 'when ENV exists' do - let(:modify_default_rspec_formatters) { 'false' } - before { stub_const("ENV", { 'KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS' => modify_default_rspec_formatters }) } - it { should eq modify_default_rspec_formatters } - end - - context "when ENV doesn't exist" do - it { should be true } - end - end - - describe '.modify_default_rspec_formatters?' do - subject { described_class.modify_default_rspec_formatters? } - - before do - expect(described_class).to receive(:modify_default_rspec_formatters).and_return(modify_default_rspec_formatters) - end - - context 'when enabled' do - let(:modify_default_rspec_formatters) { true } - - it { should be true } - end - - context 'when disabled' do - let(:modify_default_rspec_formatters) { false } - - it { should be false } - end - end - describe '.branch_encrypted' do subject { described_class.branch_encrypted } @@ -975,7 +941,7 @@ end context "when ENV doesn't exist" do - it { should eql ::Logger::DEBUG } + it { should eql ::Logger::INFO } end end diff --git a/spec/knapsack_pro/formatters/time_tracker_specs.rb b/spec/knapsack_pro/formatters/time_tracker_specs.rb index ebad0ccd..6ab89330 100644 --- a/spec/knapsack_pro/formatters/time_tracker_specs.rb +++ b/spec/knapsack_pro/formatters/time_tracker_specs.rb @@ -4,6 +4,7 @@ require 'rspec/core' require 'knapsack_pro' require 'stringio' +require 'tempfile' require_relative '../../../lib/knapsack_pro/formatters/time_tracker' class TestTimeTracker @@ -327,24 +328,6 @@ def test_duration end end - def test_batch_duration - KnapsackPro::Formatters::TimeTracker.define_method(:rspec_split_by_test_example?) do |_file| - false - end - - spec = <<~SPEC - describe "KnapsackPro::Formatters::TimeTracker" do - it do - expect(1).to eq 1 - end - end - SPEC - - run_specs(spec) do |_, _, time_tracker| - raise unless time_tracker.batch_duration > 0.0 - end - end - def test_unexecuted_test_files KnapsackPro::Formatters::TimeTracker.define_method(:rspec_split_by_test_example?) do |_file| false @@ -372,14 +355,6 @@ def test_subset KnapsackPro::Formatters::TimeTracker.define_method(:rspec_split_by_test_example?) do |_file| false end - KnapsackPro::Formatters::TimeTracker.class_eval do - alias_method :original_stop, :stop - - # In Regular Mode, #subset is called before #stop. - # This test makes #stop a noop to simulate that behavior. - define_method(:stop) do |_| - end - end spec = <<~SPEC describe "KnapsackPro::Formatters::TimeTracker" do @@ -401,23 +376,20 @@ def test_subset raise unless files[0]["time_execution"] > 0.10 raise unless files[0]["time_execution"] < 0.15 end - - ensure - KnapsackPro::Formatters::TimeTracker.class_eval do - undef :stop - alias_method :stop, :original_stop - end end private def run_specs(specs) - paths = Array(specs).map.with_index do |spec, i| - path = "spec/knapsack_pro/formatters/#{i}_#{SecureRandom.uuid}_spec.rb" - File.open(path, 'w') { |file| file.write(spec) } - path + files = Array(specs).map.with_index do |spec, i| + file = Tempfile.new(["time_tracker_#{i}", "_spec.rb"], "./spec/knapsack_pro/formatters/") + file.write(spec) + file.rewind + file end + paths = files.map(&:path).map { _1.sub("./", "") } + options = ::RSpec::Core::ConfigurationOptions.new([ "--format", KnapsackPro::Formatters::TimeTracker.to_s, *paths, @@ -436,7 +408,6 @@ def run_specs(specs) yield(paths, times, time_tracker) ensure - paths.each { |path| File.delete(path) } # Need to reset because RSpec keeps reusing the same instance. time_tracker.instance_variable_set(:@queue, {}) if time_tracker time_tracker.instance_variable_set(:@started, time_tracker.send(:now)) if time_tracker diff --git a/spec/knapsack_pro/hooks/queue_spec.rb b/spec/knapsack_pro/hooks/queue_spec.rb index d3968c37..ec335ea2 100644 --- a/spec/knapsack_pro/hooks/queue_spec.rb +++ b/spec/knapsack_pro/hooks/queue_spec.rb @@ -93,8 +93,8 @@ let(:subset_queue_id) { double } before do - expect(KnapsackPro::Config::Env).to receive(:queue_id).twice.and_return(queue_id) - expect(KnapsackPro::Config::Env).to receive(:subset_queue_id).twice.and_return(subset_queue_id) + expect(KnapsackPro::Config::Env).to receive(:queue_id).at_least(:once).and_return(queue_id) + expect(KnapsackPro::Config::Env).to receive(:subset_queue_id).at_least(:once).and_return(subset_queue_id) $expected_called_blocks = [] diff --git a/spec/knapsack_pro/presenter_spec.rb b/spec/knapsack_pro/presenter_spec.rb index 3d1978e1..eae7996d 100644 --- a/spec/knapsack_pro/presenter_spec.rb +++ b/spec/knapsack_pro/presenter_spec.rb @@ -8,7 +8,7 @@ expect(KnapsackPro).to receive(:tracker).and_return(tracker) end - it { should eql "Global time execution for tests: 01h 02m 03s" } + it { should eql "Global test execution duration: 01h 02m 03s" } end describe '.pretty_seconds' do diff --git a/spec/knapsack_pro/pure/queue/rspec_pure_spec.rb b/spec/knapsack_pro/pure/queue/rspec_pure_spec.rb new file mode 100644 index 00000000..02287212 --- /dev/null +++ b/spec/knapsack_pro/pure/queue/rspec_pure_spec.rb @@ -0,0 +1,224 @@ +require(KnapsackPro.root + '/lib/knapsack_pro/formatters/time_tracker') +require(KnapsackPro.root + '/lib/knapsack_pro/extensions/rspec_extension') + +describe KnapsackPro::Pure::Queue::RSpecPure do + let(:rspec_pure) { described_class.new } + + describe '#add_knapsack_pro_formatters_to' do + subject { rspec_pure.add_knapsack_pro_formatters_to(spec_opts) } + + context 'when no spec_opts' do + let(:spec_opts) { nil } + + it 'returns no spec_opts' do + expect(subject).to be nil + end + end + + context 'when spec_opts have Knapsack Pro formatters' do + let(:spec_opts) { '--color --format d --format KnapsackPro::Formatters::TimeTracker' } + + it 'returns spec_opts' do + expect(subject).to eq spec_opts + end + end + + context 'when spec_opts have no Knapsack Pro formatters' do + let(:spec_opts) { '--color --format d' } + + it 'returns spec_opts with added Knapsack Pro formatters' do + expect(subject).to eq '--color --format d --format KnapsackPro::Formatters::TimeTracker' + end + end + end + + describe '#error_exit_code' do + subject { rspec_pure.error_exit_code(rspec_error_exit_code) } + + context 'when RSpec has no defined error exit code' do + let(:rspec_error_exit_code) { nil } + + it 'returns 1 as the default exit code' do + expect(subject).to eq 1 + end + end + + context 'when RSpec has a defined error exit code' do + let(:rspec_error_exit_code) { 2 } + + it 'returns the custom exit code' do + expect(subject).to eq rspec_error_exit_code + end + end + end + + describe '#args_with_seed_option_added_when_viable' do + let(:order_option) { KnapsackPro::Adapters::RSpecAdapter.order_option(args) } + + subject { rspec_pure.args_with_seed_option_added_when_viable(order_option, seed, args) } + + context 'when the order option is not random' do + let(:args) { ['--order', 'defined'] } + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: nil, used?: false) } + + it 'does not add the seed option to args' do + expect(subject).to eq ['--order', 'defined'] + end + end + + ['random', 'rand'].each do |random_option_value| + context "when the order option is `#{random_option_value}`" do + let(:args) { ['--order', random_option_value] } + + context 'when the seed is not used' do + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: '123', used?: false) } + + it 'does not add the seed option to args' do + expect(subject).to eq ['--order', random_option_value] + end + end + + context 'when the seed is used' do + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: '123', used?: true) } + + it 'adds the seed option to args' do + expect(subject).to eq ['--order', random_option_value, '--seed', '123'] + end + end + end + end + + context 'when the order option is `rand:123`' do + let(:args) { ['--order', 'rand:123'] } + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: '123', used?: true) } + + it 'does not add the seed option to args' do + expect(subject).to eq ['--order', 'rand:123'] + end + end + + context 'when the order option is not set in args AND seed is used' do + let(:args) { ['--format', 'documentation'] } + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: '123', used?: true) } + + it 'adds the seed option to args' do + expect(subject).to eq ['--format', 'documentation', '--seed', '123'] + end + end + + context 'when the order option is not set in args AND seed is not used' do + let(:args) { ['--format', 'documentation'] } + let(:seed) { KnapsackPro::Extensions::RSpecExtension::Seed.new(value: '123', used?: false) } + + it 'does not add the seed option to args' do + expect(subject).to eq ['--format', 'documentation'] + end + end + end + + describe '#prepare_cli_args' do + subject { rspec_pure.prepare_cli_args(args, has_format_option, test_dir) } + + context 'when no args' do + let(:args) { nil } + let(:has_format_option) { false } + let(:test_dir) { 'spec' } + + it 'adds the default progress formatter and the default path and the time tracker formatter' do + expect(subject).to eq [ + '--format', 'progress', + '--default-path', 'spec', + '--format', 'KnapsackPro::Formatters::TimeTracker', + ] + end + end + + context 'when args are present and a custom test directory is set' do + let(:args) { '--color --profile' } + let(:has_format_option) { false } + let(:test_dir) { 'custom_spec_dir' } + + it do + expect(subject).to eq [ + '--color', + '--profile', + '--format', 'progress', + '--default-path', 'custom_spec_dir', + '--format', 'KnapsackPro::Formatters::TimeTracker', + ] + end + end + + context 'when args are present and has format option' do + let(:args) { '--color --profile --format d' } + let(:has_format_option) { true } + let(:test_dir) { 'spec' } + + it 'uses the format option from args instead of the default formatter' do + expect(subject).to eq [ + '--color', + '--profile', + '--format', 'd', + '--default-path', 'spec', + '--format', 'KnapsackPro::Formatters::TimeTracker', + ] + end + end + end + + describe '#rspec_command' do + let(:args) { ['--format', 'documentation'] } + let(:test_file_paths) { ['a_spec.rb', 'b_spec.rb'] } + + subject { rspec_pure.rspec_command(args, test_file_paths, scope) } + + context 'when there are no test file paths' do + let(:scope) { :queue_finished } + let(:test_file_paths) { [] } + + it 'returns no messages' do + expect(subject).to eq [] + end + end + + context 'when a subset of queue (a batch of tests fetched from the Queue API)' do + let(:scope) { :batch_finished } + + it 'returns messages with the RSpec command' do + expect(subject).to eq([ + 'To retry the last batch of tests fetched from the Queue API, please run the following command on your machine:', + 'bundle exec rspec --format documentation "a_spec.rb" "b_spec.rb"', + ]) + end + end + + context 'when all tests fetched from the Queue API' do + let(:scope) { :queue_finished } + + it 'returns messages with the RSpec command' do + expect(subject).to eq([ + 'To retry all the tests assigned to this CI node, please run the following command on your machine:', + 'bundle exec rspec --format documentation "a_spec.rb" "b_spec.rb"', + ]) + end + end + + describe '#exit_summary' do + subject { rspec_pure.exit_summary(unexecuted_test_files) } + + context 'when there are no unexecuted test files' do + let(:unexecuted_test_files) { [] } + + it { expect(subject).to be_nil } + end + + context 'when there are unexecuted test files' do + let(:unexecuted_test_files) { ['b_spec.rb', 'c_spec.rb'] } + + it 'returns a warning message' do + expect(subject).to eq 'Unexecuted tests on this CI node (including pending tests): b_spec.rb c_spec.rb' + end + end + end + end +end diff --git a/spec/knapsack_pro/runners/queue/cucumber_runner_spec.rb b/spec/knapsack_pro/runners/queue/cucumber_runner_spec.rb index a3b0b5bc..4a154447 100644 --- a/spec/knapsack_pro/runners/queue/cucumber_runner_spec.rb +++ b/spec/knapsack_pro/runners/queue/cucumber_runner_spec.rb @@ -24,7 +24,7 @@ expect(described_class).to receive(:new).with(KnapsackPro::Adapters::CucumberAdapter).and_return(runner) end - context 'when args provided' do + context 'when args are provided' do let(:args) { '--retry 5 --no-strict-flaky' } it do @@ -39,7 +39,7 @@ can_initialize_queue: true, args: args, exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } expect(described_class).to receive(:handle_signal!) expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) @@ -50,7 +50,7 @@ end end - context 'when args not provided' do + context 'when args are not provided' do let(:args) { nil } it do @@ -65,7 +65,7 @@ can_initialize_queue: true, args: nil, exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } expect(described_class).to receive(:handle_signal!) expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) @@ -85,21 +85,21 @@ let(:can_initialize_queue) { double(:can_initialize_queue) } let(:args) { '--retry 5 --no-strict-flaky' } let(:exitstatus) { 0 } - let(:all_test_file_paths) { [] } + let(:node_test_file_paths) { [] } let(:accumulator) do { runner: runner, can_initialize_queue: can_initialize_queue, args: args, exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, + node_test_file_paths: node_test_file_paths, } end subject { described_class.run_tests(accumulator) } before do - expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: all_test_file_paths).and_return(test_file_paths) + expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: node_test_file_paths).and_return(test_file_paths) end context 'when test files exist' do @@ -129,7 +129,7 @@ allow(child_status).to receive(:exitstatus).and_return(exitstatus) end - context 'when system process finished its work (exited)' do + context 'when system process finished (exited)' do let(:process_exited) { true } context 'when tests are passing' do @@ -142,7 +142,7 @@ can_initialize_queue: false, args: args, exitstatus: exitstatus, - all_test_file_paths: test_file_paths, + node_test_file_paths: test_file_paths, }) end end @@ -157,13 +157,13 @@ can_initialize_queue: false, args: args, exitstatus: 1, # tests failed - all_test_file_paths: test_file_paths, + node_test_file_paths: test_file_paths, }) end end end - context "when system process didn't finish its work (hasn't exited)" do + context 'when system process did not finish (it has not exited)' do let(:process_exited) { false } it 'raises an error' do @@ -172,11 +172,11 @@ end end - context "when test files don't exist" do + context 'when test files do not exist' do let(:test_file_paths) { [] } - context 'when all_test_file_paths exist' do - let(:all_test_file_paths) { ['features/a.feature'] } + context 'when node_test_file_paths exists' do + let(:node_test_file_paths) { ['features/a.feature'] } it 'returns exit code 0' do expect(KnapsackPro::Adapters::CucumberAdapter).to receive(:verify_bind_method_called) @@ -191,8 +191,8 @@ end end - context "when all_test_file_paths don't exist" do - let(:all_test_file_paths) { [] } + context 'when node_test_file_paths do not exist' do + let(:node_test_file_paths) { [] } it 'returns exit code 0' do expect(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) diff --git a/spec/knapsack_pro/runners/queue/minitest_runner_spec.rb b/spec/knapsack_pro/runners/queue/minitest_runner_spec.rb index 91fd339e..4b6e853f 100644 --- a/spec/knapsack_pro/runners/queue/minitest_runner_spec.rb +++ b/spec/knapsack_pro/runners/queue/minitest_runner_spec.rb @@ -26,7 +26,7 @@ expect($LOAD_PATH).to receive(:unshift).with(test_dir) end - context 'when args provided' do + context 'when args are provided' do let(:args) { '--verbose --pride' } it do @@ -41,7 +41,7 @@ can_initialize_queue: true, args: ['--verbose', '--pride'], exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } expect(described_class).to receive(:handle_signal!) expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) @@ -52,7 +52,7 @@ end end - context 'when args not provided' do + context 'when args are not provided' do let(:args) { nil } it do @@ -67,7 +67,7 @@ can_initialize_queue: true, args: [], exitstatus: 0, - all_test_file_paths: [], + node_test_file_paths: [], } expect(described_class).to receive(:handle_signal!) expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) @@ -84,21 +84,21 @@ let(:can_initialize_queue) { double(:can_initialize_queue) } let(:args) { ['--verbose', '--pride'] } let(:exitstatus) { 0 } - let(:all_test_file_paths) { [] } + let(:node_test_file_paths) { [] } let(:accumulator) do { runner: runner, can_initialize_queue: can_initialize_queue, args: args, exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, + node_test_file_paths: node_test_file_paths, } end subject { described_class.run_tests(accumulator) } before do - expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: all_test_file_paths).and_return(test_file_paths) + expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: node_test_file_paths).and_return(test_file_paths) end context 'when test files exist' do @@ -145,7 +145,7 @@ can_initialize_queue: false, args: args, exitstatus: exitstatus, - all_test_file_paths: test_file_paths, + node_test_file_paths: test_file_paths, }) end end @@ -160,17 +160,17 @@ can_initialize_queue: false, args: args, exitstatus: 1, # tests failed - all_test_file_paths: test_file_paths, + node_test_file_paths: test_file_paths, }) end end end - context "when test files don't exist" do + context 'when test files do not exist' do let(:test_file_paths) { [] } - context 'when all_test_file_paths exist' do - let(:all_test_file_paths) { ['a_test.rb'] } + context 'when node_test_file_paths exists' do + let(:node_test_file_paths) { ['a_test.rb'] } it 'returns exit code 0' do expect(KnapsackPro::Adapters::MinitestAdapter).to receive(:verify_bind_method_called) @@ -185,8 +185,8 @@ end end - context "when all_test_file_paths don't exist" do - let(:all_test_file_paths) { [] } + context 'when node_test_file_paths do not exist' do + let(:node_test_file_paths) { [] } it 'returns exit code 0' do expect(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) diff --git a/spec/knapsack_pro/runners/queue/rspec_runner_spec.rb b/spec/knapsack_pro/runners/queue/rspec_runner_spec.rb deleted file mode 100644 index 2c6494dd..00000000 --- a/spec/knapsack_pro/runners/queue/rspec_runner_spec.rb +++ /dev/null @@ -1,536 +0,0 @@ -describe KnapsackPro::Runners::Queue::RSpecRunner do - before do - # we don't want to modify rspec formatters because we want to see tests summary at the end - # when you run this test file or whole test suite for the knapsack_pro gem - stub_const('ENV', { 'KNAPSACK_PRO_MODIFY_DEFAULT_RSPEC_FORMATTERS' => false }) - - require KnapsackPro.root + '/lib/knapsack_pro/formatters/rspec_queue_summary_formatter' - require KnapsackPro.root + '/lib/knapsack_pro/formatters/rspec_queue_profile_formatter_extension' - require KnapsackPro.root + '/lib/knapsack_pro/formatters/time_tracker' - end - - describe '.run' do - let(:test_suite_token_rspec) { 'fake-token' } - let(:queue_id) { 'fake-queue-id' } - let(:test_dir) { 'fake-test-dir' } - let(:runner) do - instance_double(described_class, test_dir: test_dir) - end - - subject { described_class.run(args) } - - before do - expect(KnapsackPro::Config::Env).to receive(:test_suite_token_rspec).and_return(test_suite_token_rspec) - expect(KnapsackPro::Config::EnvGenerator).to receive(:set_queue_id).and_return(queue_id) - - expect(ENV).to receive(:[]=).with('KNAPSACK_PRO_TEST_SUITE_TOKEN', test_suite_token_rspec) - expect(ENV).to receive(:[]=).with('KNAPSACK_PRO_QUEUE_RECORDING_ENABLED', 'true') - expect(ENV).to receive(:[]=).with('KNAPSACK_PRO_QUEUE_ID', queue_id) - - expect(KnapsackPro::Config::Env).to receive(:set_test_runner_adapter).with(KnapsackPro::Adapters::RSpecAdapter) - - expect(described_class).to receive(:new).with(KnapsackPro::Adapters::RSpecAdapter).and_return(runner) - end - - context 'when args provided' do - context 'when format option is not provided' do - let(:args) { '--example-arg example-value' } - - it 'uses default formatter progress' do - expected_exitstatus = 0 - expected_accumulator = { - status: :completed, - exitstatus: expected_exitstatus - } - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: [ - '--example-arg', 'example-value', - '--format', 'progress', - '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', - '--format', 'KnapsackPro::Formatters::TimeTracker', - '--default-path', 'fake-test-dir', - ], - exitstatus: 0, - all_test_file_paths: [], - } - expect(described_class).to receive(:handle_signal!) - expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) - - expect(Kernel).to receive(:exit).with(expected_exitstatus) - - subject - end - end - - context 'when format option is provided as --format' do - let(:args) { '--format documentation' } - - it 'uses provided format option instead of default formatter progress' do - expected_exitstatus = 0 - expected_accumulator = { - status: :completed, - exitstatus: expected_exitstatus - } - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: [ - '--format', 'documentation', - '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', - '--format', 'KnapsackPro::Formatters::TimeTracker', - '--default-path', 'fake-test-dir', - ], - exitstatus: 0, - all_test_file_paths: [], - } - expect(described_class).to receive(:handle_signal!) - expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) - - expect(Kernel).to receive(:exit).with(expected_exitstatus) - - subject - end - end - - context 'when format option is provided as -f' do - let(:args) { '-f d' } - - it 'uses provided format option instead of default formatter progress' do - expected_exitstatus = 0 - expected_accumulator = { - status: :completed, - exitstatus: expected_exitstatus - } - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: [ - '-f', 'd', - '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', - '--format', 'KnapsackPro::Formatters::TimeTracker', - '--default-path', 'fake-test-dir', - ], - exitstatus: 0, - all_test_file_paths: [], - } - expect(described_class).to receive(:handle_signal!) - expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) - - expect(Kernel).to receive(:exit).with(expected_exitstatus) - - subject - end - end - - context 'when format option is provided without a delimiter' do - let(:args) { '-fMyCustomFormatter' } - - it 'uses provided format option instead of default formatter progress' do - expected_exitstatus = 0 - expected_accumulator = { - status: :completed, - exitstatus: expected_exitstatus - } - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: [ - '-fMyCustomFormatter', - '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', - '--format', 'KnapsackPro::Formatters::TimeTracker', - '--default-path', 'fake-test-dir', - ], - exitstatus: 0, - all_test_file_paths: [], - } - expect(described_class).to receive(:handle_signal!) - expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) - - expect(Kernel).to receive(:exit).with(expected_exitstatus) - - subject - end - end - - context 'when RSpec split by test examples feature is enabled' do - before do - expect(KnapsackPro::Config::Env).to receive(:rspec_split_by_test_examples?).and_return(true) - expect(KnapsackPro::Adapters::RSpecAdapter).to receive(:ensure_no_tag_option_when_rspec_split_by_test_examples_enabled!).and_call_original - end - - context 'when tag option is provided' do - let(:args) { '--tag example-value' } - - it do - expect { subject }.to raise_error(/It is not allowed to use the RSpec tag option together with the RSpec split by test examples feature/) - end - end - end - end - - context 'when args not provided' do - let(:args) { nil } - - it do - expected_exitstatus = 0 - expected_accumulator = { - status: :completed, - exitstatus: expected_exitstatus - } - accumulator = { - status: :next, - runner: runner, - can_initialize_queue: true, - args: [ - '--format', 'progress', - '--format', 'KnapsackPro::Formatters::RSpecQueueSummaryFormatter', - '--format', 'KnapsackPro::Formatters::TimeTracker', - '--default-path', 'fake-test-dir', - ], - exitstatus: 0, - all_test_file_paths: [], - } - expect(described_class).to receive(:handle_signal!) - expect(described_class).to receive(:run_tests).with(accumulator).and_return(expected_accumulator) - - expect(Kernel).to receive(:exit).with(expected_exitstatus) - - subject - end - end - end - - describe '.run_tests' do - let(:runner) { instance_double(described_class) } - let(:can_initialize_queue) { double(:can_initialize_queue) } - let(:args) { ['--no-color', '--default-path', 'fake-test-dir'] } - let(:exitstatus) { double } - let(:all_test_file_paths) { [] } - let(:accumulator) do - { - runner: runner, - can_initialize_queue: can_initialize_queue, - args: args, - exitstatus: exitstatus, - all_test_file_paths: all_test_file_paths, - } - end - - subject { described_class.run_tests(accumulator) } - - before do - expect(runner).to receive(:test_file_paths).with(can_initialize_queue: can_initialize_queue, executed_test_files: all_test_file_paths).and_return(test_file_paths) - end - - context 'when test files exist' do - let(:test_file_paths) { ['a_spec.rb', 'b_spec.rb'] } - let(:logger) { double } - let(:rspec_seed) { 7771 } - let(:exit_code) { [0, 1].sample } - let(:rspec_wants_to_quit) { false } - let(:rspec_is_quitting) { false } - let(:rspec_core_runner) do - double(world: double(wants_to_quit: rspec_wants_to_quit, rspec_is_quitting: rspec_is_quitting)) - end - - context 'having no exception when running RSpec' do - before do - subset_queue_id = 'fake-subset-queue-id' - expect(KnapsackPro::Config::EnvGenerator).to receive(:set_subset_queue_id).and_return(subset_queue_id) - - expect(ENV).to receive(:[]=).with('KNAPSACK_PRO_SUBSET_QUEUE_ID', subset_queue_id) - - expect(described_class).to receive(:ensure_spec_opts_have_knapsack_pro_formatters) - options = double - expect(RSpec::Core::ConfigurationOptions).to receive(:new).with([ - '--no-color', - '--default-path', 'fake-test-dir', - 'a_spec.rb', 'b_spec.rb', - ]).and_return(options) - - expect(RSpec::Core::Runner).to receive(:new).with(options).and_return(rspec_core_runner) - expect(rspec_core_runner).to receive(:run).with($stderr, $stdout).and_return(exit_code) - - expect(described_class).to receive(:rspec_clear_examples) - - expect(KnapsackPro::Hooks::Queue).to receive(:call_before_subset_queue) - - expect(KnapsackPro::Hooks::Queue).to receive(:call_after_subset_queue) - - configuration = double - expect(rspec_core_runner).to receive(:configuration).twice.and_return(configuration) - expect(configuration).to receive(:seed_used?).and_return(true) - expect(configuration).to receive(:seed).and_return(rspec_seed) - - expect(KnapsackPro).to receive(:logger).at_least(2).and_return(logger) - expect(logger).to receive(:info) - .with("To retry the last batch of tests fetched from the API Queue, please run the following command on your machine:") - expect(logger).to receive(:info).with(/#{args.join(' ')} --seed #{rspec_seed}/) - end - - context 'when the exit code is zero' do - let(:exit_code) { 0 } - - it do - expect(subject).to eq({ - status: :next, - runner: runner, - can_initialize_queue: false, - args: args, - exitstatus: exitstatus, - all_test_file_paths: test_file_paths, - }) - end - end - - context 'when the exit code is not zero' do - let(:exit_code) { double } - - it do - expect(subject).to eq({ - status: :next, - runner: runner, - can_initialize_queue: false, - args: args, - exitstatus: exit_code, - all_test_file_paths: test_file_paths, - }) - end - end - - context 'when RSpec wants to quit' do - let(:exit_code) { 0 } - let(:rspec_wants_to_quit) { true } - - after do - described_class.class_variable_set(:@@terminate_process, false) - end - - it 'terminates the process' do - expect(logger).to receive(:warn).with('RSpec wants to quit.') - - expect(described_class.class_variable_get(:@@terminate_process)).to be false - - expect(subject).to eq({ - status: :next, - runner: runner, - can_initialize_queue: false, - args: args, - exitstatus: exitstatus, - all_test_file_paths: test_file_paths, - }) - - expect(described_class.class_variable_get(:@@terminate_process)).to be true - end - end - - context 'when RSpec is quitting' do - let(:exit_code) { 0 } - let(:rspec_is_quitting) { true } - - after do - described_class.class_variable_set(:@@terminate_process, false) - end - - it 'terminates the process' do - expect(logger).to receive(:warn).with('RSpec is quitting.') - - expect(described_class.class_variable_get(:@@terminate_process)).to be false - - expect(subject).to eq({ - status: :next, - runner: runner, - can_initialize_queue: false, - args: args, - exitstatus: exitstatus, - all_test_file_paths: test_file_paths, - }) - - expect(described_class.class_variable_get(:@@terminate_process)).to be true - end - end - end - - context 'having exception when running RSpec' do - before do - subset_queue_id = 'fake-subset-queue-id' - expect(KnapsackPro::Config::EnvGenerator).to receive(:set_subset_queue_id).and_return(subset_queue_id) - - expect(ENV).to receive(:[]=).with('KNAPSACK_PRO_SUBSET_QUEUE_ID', subset_queue_id) - - expect(described_class).to receive(:ensure_spec_opts_have_knapsack_pro_formatters) - options = double - expect(RSpec::Core::ConfigurationOptions).to receive(:new).with([ - '--no-color', - '--default-path', 'fake-test-dir', - 'a_spec.rb', 'b_spec.rb', - ]).and_return(options) - - rspec_core_runner = double - expect(RSpec::Core::Runner).to receive(:new).with(options).and_return(rspec_core_runner) - expect(rspec_core_runner).to receive(:run).with($stderr, $stdout).and_raise SystemExit - expect(KnapsackPro::Hooks::Queue).to receive(:call_before_subset_queue) - allow(KnapsackPro::Hooks::Queue).to receive(:call_after_subset_queue) - allow(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) - allow(KnapsackPro::Formatters::RSpecQueueSummaryFormatter).to receive(:print_exit_summary) - expect(Kernel).to receive(:exit).with(1) - end - - it 'does not call #rspec_clear_examples' do - expect(described_class).not_to receive(:rspec_clear_examples) - expect { subject }.to raise_error SystemExit - end - - it 'logs the exception' do - expect(KnapsackPro).to receive(:logger).once.and_return(logger) - expect(logger).to receive(:error).with("Having exception when running RSpec: #") - expect { subject }.to raise_error SystemExit - end - - it 'calls #print_exit_summary' do - expect(KnapsackPro::Formatters::RSpecQueueSummaryFormatter).to receive(:print_exit_summary) - expect { subject }.to raise_error SystemExit - end - - it 'calls #call_after_subset_queue and #call_after_queue' do - expect(KnapsackPro::Hooks::Queue).to receive(:call_after_subset_queue) - expect(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) - expect { subject }.to raise_error SystemExit - end - end - end - - context "when test files don't exist" do - let(:test_file_paths) { [] } - - context 'when all_test_file_paths exist' do - let(:all_test_file_paths) { ['a_spec.rb'] } - let(:logger) { double } - - before do - described_class.class_variable_set(:@@used_seed, used_seed) - - expect(KnapsackPro).to receive(:logger).twice.and_return(logger) - - expect(KnapsackPro::Adapters::RSpecAdapter).to receive(:verify_bind_method_called) - - expect(KnapsackPro::Formatters::RSpecQueueSummaryFormatter).to receive(:print_summary) - expect(KnapsackPro::Formatters::RSpecQueueProfileFormatterExtension).to receive(:print_summary) - - expect(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) - - time_tracker = instance_double(KnapsackPro::Formatters::TimeTracker) - times = all_test_file_paths.map do |path| - [{ path: path, time_execution: 1.0 }] - end - expect(time_tracker).to receive(:queue).and_return(times) - expect(KnapsackPro::Formatters::TimeTrackerFetcher).to receive(:call).and_return(time_tracker) - expect(KnapsackPro::Report).to receive(:save_node_queue_to_api).with(times) - - expect(logger).to receive(:info) - .with('To retry all the tests assigned to this CI node, please run the following command on your machine:') - expect(logger).to receive(:info).with(logged_rspec_command_matcher) - end - - context 'when @@used_seed has been set' do - let(:used_seed) { '8333' } - let(:logged_rspec_command_matcher) { /#{args.join(' ')} --seed #{used_seed} \"a_spec.rb"/ } - - it do - expect(subject).to eq({ - status: :completed, - exitstatus: exitstatus, - }) - end - end - - context 'when @@used_seed has not been set' do - let(:used_seed) { nil } - let(:logged_rspec_command_matcher) { /#{args.join(' ')} \"a_spec.rb"/ } - - it do - expect(subject).to eq({ - status: :completed, - exitstatus: exitstatus, - }) - end - end - end - - context "when all_test_file_paths don't exist" do - let(:all_test_file_paths) { [] } - - it do - expect(KnapsackPro::Hooks::Queue).to receive(:call_after_queue) - - time_tracker = instance_double(KnapsackPro::Formatters::TimeTracker) - times = all_test_file_paths.map do |path| - [{ path: path, time_execution: 0.0 }] - end - expect(time_tracker).to receive(:queue).and_return(times) - expect(KnapsackPro::Formatters::TimeTrackerFetcher).to receive(:call).and_return(time_tracker) - expect(KnapsackPro::Report).to receive(:save_node_queue_to_api).with(times) - - expect(KnapsackPro).to_not receive(:logger) - - expect(subject).to eq({ - status: :completed, - exitstatus: exitstatus, - }) - end - end - end - end - - describe '.ensure_spec_opts_have_knapsack_pro_formatters' do - subject { described_class.ensure_spec_opts_have_knapsack_pro_formatters } - - context 'when `SPEC_OPTS` is set' do - context 'when `SPEC_OPTS` has RSpecQueueSummaryFormatter' do - before do - stub_const('ENV', { 'SPEC_OPTS' => '--format json --format KnapsackPro::Formatters::RSpecQueueSummaryFormatter' }) - end - - it 'adds TimeTracker' do - subject - expect(ENV['SPEC_OPTS']).to eq '--format json --format KnapsackPro::Formatters::RSpecQueueSummaryFormatter --format KnapsackPro::Formatters::TimeTracker' - end - end - - context 'when `SPEC_OPTS` has TimeTracker' do - before do - stub_const('ENV', { 'SPEC_OPTS' => '--format json --format KnapsackPro::Formatters::TimeTracker' }) - end - - it 'adds RSpecQueueSummaryFormatter' do - subject - expect(ENV['SPEC_OPTS']).to eq '--format json --format KnapsackPro::Formatters::TimeTracker --format KnapsackPro::Formatters::RSpecQueueSummaryFormatter' - end - end - - context 'when `SPEC_OPTS` has no Knapsack Pro formatters' do - before do - stub_const('ENV', { 'SPEC_OPTS' => '--format json' }) - end - - it 'adds RSpecQueueSummaryFormatter and TimeTracker to `SPEC_OPTS`' do - subject - expect(ENV['SPEC_OPTS']).to eq '--format json --format KnapsackPro::Formatters::RSpecQueueSummaryFormatter --format KnapsackPro::Formatters::TimeTracker' - end - end - end - - context 'when `SPEC_OPTS` is not set' do - it 'does nothing' do - subject - expect(ENV['SPEC_OPTS']).to be_nil - end - end - end -end diff --git a/spec/knapsack_pro_spec.rb b/spec/knapsack_pro_spec.rb index c81751e1..8c9cad6c 100644 --- a/spec/knapsack_pro_spec.rb +++ b/spec/knapsack_pro_spec.rb @@ -24,7 +24,7 @@ }) expect(Logger).to receive(:new).with('log/knapsack_pro_node_1.log').and_return(logger) - expect(logger).to receive(:level=).with(Logger::DEBUG) + expect(logger).to receive(:level=).with(Logger::INFO) expect(KnapsackPro::LoggerWrapper).to receive(:new).with(logger).and_return(logger_wrapper) end @@ -38,7 +38,7 @@ }) expect(Logger).to receive(:new).with('log/knapsack_pro_node_0.log').and_return(logger) - expect(logger).to receive(:level=).with(Logger::DEBUG) + expect(logger).to receive(:level=).with(Logger::INFO) expect(KnapsackPro::LoggerWrapper).to receive(:new).with(logger).and_return(logger_wrapper) end @@ -51,7 +51,7 @@ before do expect(Logger).to receive(:new).with(STDOUT).and_return(logger) - expect(logger).to receive(:level=).with(Logger::DEBUG) + expect(logger).to receive(:level=).with(Logger::INFO) expect(KnapsackPro::LoggerWrapper).to receive(:new).with(logger).and_return(logger_wrapper) end