diff --git a/features/command_line/only_failures.feature b/features/command_line/only_failures.feature index 87b7bb4c51..0ad5c9e2be 100644 --- a/features/command_line/only_failures.feature +++ b/features/command_line/only_failures.feature @@ -110,4 +110,4 @@ Feature: Using the `--only-failures` option Scenario: Clear error given when using `--only-failures` without configuring `example_status_persistence_file_path` Given I have not configured `example_status_persistence_file_path` When I run `rspec --only-failures` - Then it should fail with "To use `--only-failures`, you must first set `config.example_status_persistence_file_path`." + Then it should fail with "To use `--only-failures` or `--only-pending`, you must first set `config.example_status_persistence_file_path`." diff --git a/features/command_line/only_pending.feature b/features/command_line/only_pending.feature new file mode 100644 index 0000000000..823ce87f84 --- /dev/null +++ b/features/command_line/only_pending.feature @@ -0,0 +1,94 @@ +Feature: Using the `--only-pending` option + + The `--only-pending` option filters what examples are run so that only those that failed the last time they ran are executed. To use this option, you first have to configure `config.example_status_persistence_file_path`, which RSpec will use to store the status of each example the last time it ran. + + Either of these options can be combined with another a directory or file name; RSpec will run just the failures from the set of loaded examples. + + Background: + Given a file named "spec/spec_helper.rb" with: + """ruby + RSpec.configure do |c| + c.example_status_persistence_file_path = "examples.txt" + end + """ + And a file named ".rspec" with: + """ + --require spec_helper + --order random + --format documentation + """ + And a file named "spec/array_spec.rb" with: + """ruby + RSpec.describe 'Array' do + it "checks for inclusion of 1" do + expect([1, 2]).to include(1) + end + + it "checks for inclusion of 2", skip: "just not ready for this yet..." do + expect([1, 2]).to include(2) + end + + it "checks for inclusion of 3" do + expect([1, 2]).to include(3) # failure + end + end + """ + And a file named "spec/string_spec.rb" with: + """ruby + RSpec.describe 'String' do + it "checks for inclusion of 'foo'" do + expect("food").to include('foo') + end + + it "checks for inclusion of 'bar'" do + expect("food").to include('bar') # failure + end + + it "checks for inclusion of 'baz'" do + expect("bazzy").to include('baz') + end + + it "checks for inclusion of 'foobar'" do + expect("food").to include('foobar') # failure + end + + it "checks for inclusion of 'sum'", skip: "just not ready for this yet..." do + expect("lorem ipsum").to include('sum') + end + + it "checks for inclusion of 'sit'", skip: "...nor am I ready for this..." do + expect("dolor sit").to include('sit') + end + end + """ + And a file named "spec/passing_spec.rb" with: + """ruby + puts "Loading passing_spec.rb" + + RSpec.describe "A passing spec" do + it "passes" do + expect(1).to eq(1) + end + end + """ + And I have run `rspec` once, resulting in "10 examples, 3 failures, 3 pending" + + Scenario: Running `rspec --only-pending` loads only spec files with failures and runs only the failures + When I run `rspec --only-pending` + Then the output from "rspec --only-pending" should contain "3 examples, 0 failures, 3 pending" + And the output from "rspec --only-pending" should not contain "Loading passing_spec.rb" + + Scenario: Combine `--only-pending` with a file name + When I run `rspec spec/array_spec.rb --only-pending` + Then the output should contain "1 example, 0 failures, 1 pending" + When I run `rspec spec/string_spec.rb --only-pending` + Then the output should contain "2 examples, 0 failures, 2 pending" + + Scenario: Running `rspec --only-pending` with spec files that pass doesn't run anything + When I run `rspec spec/passing_spec.rb --only-pending` + Then it should pass with "0 examples, 0 failures" + + Scenario: Clear error given when using `--only-pending` without configuring `example_status_persistence_file_path` + Given I have not configured `example_status_persistence_file_path` + When I run `rspec --only-pending` + Then it should fail with "To use `--only-failures` or `--only-pending`, you must first set `config.example_status_persistence_file_path`." diff --git a/lib/rspec/core/configuration.rb b/lib/rspec/core/configuration.rb index 446e21321d..91ea44e1a2 100644 --- a/lib/rspec/core/configuration.rb +++ b/lib/rspec/core/configuration.rb @@ -179,7 +179,7 @@ def deprecation_stream=(value) # @macro define_reader # The file path to use for persisting example statuses. Necessary for the - # `--only-failures` and `--next-failure` CLI options. + # `--only-failures`, `--only-pending`, and `--next-failure` CLI options. # # @overload example_status_persistence_file_path # @return [String] the file path @@ -188,7 +188,7 @@ def deprecation_stream=(value) define_reader :example_status_persistence_file_path # Sets the file path to use for persisting example statuses. Necessary for the - # `--only-failures` and `--next-failure` CLI options. + # `--only-failures`, `--only-pending`, and `--next-failure` CLI options. def example_status_persistence_file_path=(value) @example_status_persistence_file_path = value clear_values_derived_from_example_status_persistence_file_path @@ -199,9 +199,24 @@ def example_status_persistence_file_path=(value) define_reader :only_failures alias_method :only_failures?, :only_failures + # @macro define_reader + # Indicates if the `--only-pending` flag is being used. + define_reader :only_pending + alias_method :only_pending?, :only_pending + + # @private + def only_flag_set? + only_failures? || only_pending? + end + # @private - def only_failures_but_not_configured? - only_failures? && !example_status_persistence_file_path + def multiple_only_flags? + only_failures && only_pending? + end + + # @private + def only_flag_but_not_configured? + only_flag_set? && !example_status_persistence_file_path end # @macro define_reader @@ -1143,6 +1158,14 @@ def spec_files_with_failures end.to_a end + # @private + def spec_files_with_pending + @spec_files_with_pending ||= last_run_statuses.inject(Set.new) do |files, (id, status)| + files << Example.parse_id(id).first if status == PENDING_STATUS + files + end.to_a + end + # Creates a method that delegates to `example` including the submitted # `args`. Used internally to add variants of `example` like `pending`: # @param name [String] example name alias @@ -2196,15 +2219,27 @@ def run_suite_hooks(hook_description, hooks) end end - def get_files_to_run(paths) - files = FlatMap.flat_map(paths_to_check(paths)) do |path| + def get_files(paths) + FlatMap.flat_map(paths_to_check(paths)) do |path| path = path.gsub(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR File.directory?(path) ? gather_directories(path) : extract_location(path) end.uniq + end + + def get_files_to_run(paths) + files = get_files(paths) + return files unless only_flag_set? - return files unless only_failures? relative_files = files.map { |f| Metadata.relative_path(File.expand_path f) } - intersection = (relative_files & spec_files_with_failures.to_a) + + # If both are set, #fail_if_config_and_cli_options_invalid should have caught it. + case [only_failures?, only_pending?] + in [true, _] + intersection = (relative_files & spec_files_with_failures.to_a) + in [_, true] + intersection = (relative_files & spec_files_with_pending.to_a) + end + intersection.empty? ? files : intersection end @@ -2339,6 +2374,7 @@ def update_pattern_attr(name, value) def clear_values_derived_from_example_status_persistence_file_path @last_run_statuses = nil @spec_files_with_failures = nil + @spec_files_with_pending = nil end def configure_group_with(group, module_list, application_method) diff --git a/lib/rspec/core/configuration_options.rb b/lib/rspec/core/configuration_options.rb index 01325088e4..02cdf40db6 100644 --- a/lib/rspec/core/configuration_options.rb +++ b/lib/rspec/core/configuration_options.rb @@ -89,7 +89,7 @@ def order(keys) # `files_or_directories_to_run` uses `default_path` so it must be # set before it. - :default_path, :only_failures, + :default_path, :only_failures, :only_pending, # These must be set before `requires` to support checking # `config.files_to_run` from within `spec_helper.rb` when a diff --git a/lib/rspec/core/option_parser.rb b/lib/rspec/core/option_parser.rb index 35ed0c9501..b402984c96 100644 --- a/lib/rspec/core/option_parser.rb +++ b/lib/rspec/core/option_parser.rb @@ -207,16 +207,20 @@ def parser(options) FILTERING parser.on('--only-failures', "Filter to just the examples that failed the last time they ran.") do - configure_only_failures(options) + configure_only(options, :only_failures, 'failed') end parser.on("-n", "--next-failure", "Apply `--only-failures` and abort after one failure.", " (Equivalent to `--only-failures --fail-fast --order defined`)") do - configure_only_failures(options) + configure_only(options, :only_failures, 'failed') set_fail_fast(options, 1) options[:order] ||= 'defined' end + parser.on('--only-pending', "Filter to just the examples that were pending the last time they ran.") do + configure_only(options, :only_pending, 'pending') + end + parser.on('-P', '--pattern PATTERN', 'Load files matching pattern (default: "spec/**/*_spec.rb").') do |o| if options[:pattern] options[:pattern] += ',' + o @@ -315,9 +319,9 @@ def set_fail_fast(options, value) options[:fail_fast] = value end - def configure_only_failures(options) - options[:only_failures] = true - add_tag_filter(options, :inclusion_filter, :last_run_status, 'failed') + def configure_only(options, type, value) + options[type] = true + add_tag_filter(options, :inclusion_filter, :last_run_status, value) end end end diff --git a/lib/rspec/core/project_initializer/spec/spec_helper.rb b/lib/rspec/core/project_initializer/spec/spec_helper.rb index c80d44b974..27d4a4c952 100644 --- a/lib/rspec/core/project_initializer/spec/spec_helper.rb +++ b/lib/rspec/core/project_initializer/spec/spec_helper.rb @@ -55,8 +55,8 @@ config.filter_run_when_matching :focus # Allows RSpec to persist some state between runs in order to support - # the `--only-failures` and `--next-failure` CLI options. We recommend - # you configure your source control system to ignore this file. + # the `--only-failures`, `--only-pending`, and `--next-failure` CLI options. + # We recommend you configure your source control system to ignore this file. config.example_status_persistence_file_path = "spec/examples.txt" # Limits the available syntax to the non-monkey patched syntax that is diff --git a/lib/rspec/core/world.rb b/lib/rspec/core/world.rb index 6fb439659b..0d8484c9cc 100644 --- a/lib/rspec/core/world.rb +++ b/lib/rspec/core/world.rb @@ -183,7 +183,7 @@ def announce_filters end end - if @configuration.run_all_when_everything_filtered? && example_count.zero? && !@configuration.only_failures? + if @configuration.run_all_when_everything_filtered? && example_count.zero? && !@configuration.only_flag_set? report_filter_message("#{everything_filtered_message}; ignoring #{inclusion_filter.description}") filtered_examples.clear inclusion_filter.clear @@ -250,13 +250,21 @@ def descending_declaration_line_numbers_by_file end def fail_if_config_and_cli_options_invalid - return unless @configuration.only_failures_but_not_configured? + if @configuration.only_flag_but_not_configured? + reporter.abort_with( + "\nTo use `--only-failures` or `--only-pending`, you must first set " \ + "`config.example_status_persistence_file_path`.", + 1 # exit code + ) + end - reporter.abort_with( - "\nTo use `--only-failures`, you must first set " \ - "`config.example_status_persistence_file_path`.", - 1 # exit code - ) + if @configuration.multiple_only_flags? + reporter.abort_with( + "\nYou cannot use `--only-failures` and `--only-pending` together. " \ + "Please set one or the other.", + 1 # exit code + ) + end end # @private diff --git a/spec/rspec/core/configuration/only_pending_support_spec.rb b/spec/rspec/core/configuration/only_pending_support_spec.rb new file mode 100644 index 0000000000..8c466b8cd9 --- /dev/null +++ b/spec/rspec/core/configuration/only_pending_support_spec.rb @@ -0,0 +1,235 @@ +module RSpec::Core + RSpec.describe Configuration, "--only-pending support" do + let(:config) { Configuration.new } + + def simulate_persisted_examples(*examples) + config.example_status_persistence_file_path = "examples.txt" + persister = class_double(ExampleStatusPersister).as_stubbed_const + + allow(persister).to receive(:load_from).with("examples.txt").and_return(examples.flatten) + end + + describe "#last_run_statuses" do + def last_run_statuses + config.last_run_statuses + end + + context "when `example_status_persistence_file_path` is configured" do + before do + simulate_persisted_examples( + { :example_id => "id_1", :status => "passed" }, + { :example_id => "id_2", :status => "failed" }, + { :example_id => "id_3", :status => "pending" } + ) + end + + it 'gets the last run statuses from the ExampleStatusPersister' do + expect(last_run_statuses).to eq( + 'id_1' => 'passed', 'id_2' => 'failed', 'id_3' => 'pending' + ) + end + + it 'returns a memoized value' do + expect(last_run_statuses).to be(last_run_statuses) + end + + specify 'the hash returns `unknown` for unknown example ids for consistency' do + expect(last_run_statuses["foo"]).to eq(Configuration::UNKNOWN_STATUS) + expect(last_run_statuses["bar"]).to eq(Configuration::UNKNOWN_STATUS) + end + end + + context "when `example_status_persistence_file_path` is not configured" do + before do + config.example_status_persistence_file_path = nil + end + + it 'returns a memoized value' do + expect(last_run_statuses).to be(last_run_statuses) + end + + it 'returns a blank hash without attempting to load the persisted statuses' do + persister = class_double(ExampleStatusPersister).as_stubbed_const + expect(persister).not_to receive(:load_from) + + expect(last_run_statuses).to eq({}) + end + + specify 'the hash returns `unknown` for all ids for consistency' do + expect(last_run_statuses["foo"]).to eq(Configuration::UNKNOWN_STATUS) + expect(last_run_statuses["bar"]).to eq(Configuration::UNKNOWN_STATUS) + end + end + + def allows_value_to_change_when_updated + simulate_persisted_examples( + { :example_id => "id_1", :status => "passed" }, + { :example_id => "id_2", :status => "failed" }, + { :example_id => "id_3", :status => "pending" } + ) + + config.example_status_persistence_file_path = nil + + expect { + yield + }.to change { last_run_statuses }.to('id_1' => 'passed', 'id_2' => 'failed', 'id_3' => 'pending') + end + + it 'allows the value to be updated when `example_status_persistence_file_path` is set after first access' do + allows_value_to_change_when_updated do + config.example_status_persistence_file_path = "examples.txt" + end + end + + it 'allows the value to be updated when `example_status_persistence_file_path` is forced after first access' do + allows_value_to_change_when_updated do + config.force(:example_status_persistence_file_path => "examples.txt") + end + end + end + + describe "#spec_files_with_pending" do + def spec_files_with_pending + config.spec_files_with_pending + end + + context "when `example_status_persistence_file_path` is configured" do + it 'returns a memoized array of unique spec files that contain failed examples' do + simulate_persisted_examples( + { :example_id => "./spec_1.rb[1:1]", :status => "failed" }, + { :example_id => "./spec_1.rb[1:2]", :status => "pending" }, + { :example_id => "./spec_1.rb[1:3]", :status => "failed" }, + { :example_id => "./spec_2.rb[1:2]", :status => "passed" }, + { :example_id => "./spec_3.rb[1:2]", :status => "pending" }, + { :example_id => "./spec_4.rb[1:2]", :status => "unknown" }, + { :example_id => "./spec_5.rb[1:2]", :status => "failed" } + ) + + expect(spec_files_with_pending).to( + be_an(Array) & + be(spec_files_with_pending) & + contain_exactly("./spec_1.rb", "./spec_3.rb") + ) + end + end + + context 'when the file at `example_status_persistence_file_path` has corrupted `status` values' do + before do + simulate_persisted_examples( + { :example_id => "./spec_1.rb[1:1]" }, + { :example_id => "./spec_1.rb[1:2]", :status => "" }, + { :example_id => "./spec_2.rb[1:2]", :status => nil }, + { :example_id => "./spec_3.rb[1:2]", :status => "wrong" }, + { :example_id => "./spec_4.rb[1:2]", :status => "unknown" }, + { :example_id => "./spec_5.rb[1:2]", :status => "failed" }, + { :example_id => "./spec_6.rb[1:2]", :status => "pending" }, + :example_id => "./spec_7.rb[1:2]", :status => "passed" + ) + end + + it 'defaults invalid statuses to unknown' do + expect(spec_files_with_pending).to( + be_an(Array) & + contain_exactly("./spec_6.rb") + ) + # Check that each example has the correct status + expect(config.last_run_statuses).to eq( + './spec_1.rb[1:1]' => 'unknown', + './spec_1.rb[1:2]' => 'unknown', + './spec_2.rb[1:2]' => 'unknown', + './spec_3.rb[1:2]' => 'unknown', + './spec_4.rb[1:2]' => 'unknown', + './spec_5.rb[1:2]' => 'failed', + './spec_6.rb[1:2]' => 'pending', + './spec_7.rb[1:2]' => 'passed' + ) + end + end + + context "when `example_status_persistence_file_path` is not configured" do + it "returns a memoized blank array" do + config.example_status_persistence_file_path = nil + + expect(spec_files_with_pending).to( + eq([]) & be(spec_files_with_pending) + ) + end + end + + def allows_value_to_change_when_updated + simulate_persisted_examples({ :example_id => "./spec_1.rb[1:1]", :status => "pending" }) + + config.example_status_persistence_file_path = nil + + expect { + yield + }.to change { spec_files_with_pending }.to(["./spec_1.rb"]) + end + + it 'allows the value to be updated when `example_status_persistence_file_path` is set after first access' do + allows_value_to_change_when_updated do + config.example_status_persistence_file_path = "examples.txt" + end + end + + it 'allows the value to be updated when `example_status_persistence_file_path` is forced after first access' do + allows_value_to_change_when_updated do + config.force(:example_status_persistence_file_path => "examples.txt") + end + end + end + + describe "#files_to_run, when `only_pending` is set" do + around do |ex| + handle_current_dir_change do + Dir.chdir("spec/rspec/core", &ex) + end + end + + let(:default_path) { "resources" } + let(:files_with_pending) { ["./resources/a_spec.rb"] } + let(:files_loaded_via_default_path) do + configuration = Configuration.new + configuration.default_path = default_path + configuration.files_or_directories_to_run = [] + configuration.files_to_run + end + + before do + expect(files_loaded_via_default_path).not_to eq(files_with_pending) + config.default_path = default_path + + simulate_persisted_examples(files_with_pending.map do |file| + { :example_id => "#{file}[1:1]", :status => "pending" } + end) + + config.force(:only_pending => true) + end + + context "and no explicit paths have been set" do + it 'loads only the files that have pending' do + config.files_or_directories_to_run = [] + expect(config.files_to_run).to eq(files_with_pending) + end + + it 'loads the default path if there are no files with pending' do + simulate_persisted_examples([]) + config.files_or_directories_to_run = [] + expect(config.files_to_run).to eq(files_loaded_via_default_path) + end + end + + context "and a path has been set" do + it "loads the intersection of files matching the path and files with pending" do + config.files_or_directories_to_run = ["resources"] + expect(config.files_to_run).to eq(files_with_pending) + end + + it "loads all files matching the path when there are no intersecting files" do + config.files_or_directories_to_run = ["resources/acceptance"] + expect(config.files_to_run).to contain_files("resources/acceptance/foo_spec.rb") + end + end + end + end +end diff --git a/spec/rspec/core/configuration_options_spec.rb b/spec/rspec/core/configuration_options_spec.rb index 27c0c836a3..9b0f669f53 100644 --- a/spec/rspec/core/configuration_options_spec.rb +++ b/spec/rspec/core/configuration_options_spec.rb @@ -124,6 +124,13 @@ opts.configure(config) end + it 'configures `only_pending` before `files_or_directories_to_run` since it affects loaded files' do + opts = config_options_object(*%w[ --only-pending ]) + expect(config).to receive(:force).with({:only_pending => true}).ordered + expect(config).to receive(:files_or_directories_to_run=).ordered + opts.configure(config) + end + { "pattern" => :pattern, "exclude-pattern" => :exclude_pattern }.each do |flag, attr| it "sets #{attr} before `requires` so users can check `files_to_run` in a `spec_helper` loaded by `--require`" do opts = config_options_object(*%W[--require spec_helper --#{flag} **/*.spec]) diff --git a/spec/rspec/core/option_parser_spec.rb b/spec/rspec/core/option_parser_spec.rb index af3edf4ec6..fda96ef1d3 100644 --- a/spec/rspec/core/option_parser_spec.rb +++ b/spec/rspec/core/option_parser_spec.rb @@ -206,6 +206,15 @@ module RSpec::Core end end + describe "--only-pending" do + it 'is equivalent to `--tag last_run_status:pending`' do + tag = Parser.parse(%w[ --tag last_run_status:pending ]) + only_failures = Parser.parse(%w[ --only-pending ]) + + expect(only_failures).to include(tag) + end + end + %w[--example -e].each do |option| describe option do it "escapes the arg" do diff --git a/spec/rspec/core/world_spec.rb b/spec/rspec/core/world_spec.rb index 7c0b52a3e4..65cdef01b9 100644 --- a/spec/rspec/core/world_spec.rb +++ b/spec/rspec/core/world_spec.rb @@ -267,6 +267,59 @@ def preceding_declaration_line(line_num) end end + context "when --only-pending is passed" do + before { configuration.force(:only_pending => true) } + + context "and all examples are filtered out" do + before do + configuration.filter_run_including :foo => 'bar' + end + + it 'will ignore run_all_when_everything_filtered' do + configuration.run_all_when_everything_filtered = true + expect(world.filtered_examples).to_not receive(:clear) + expect(world.inclusion_filter).to_not receive(:clear) + world.announce_filters + end + end + + context "and `example_status_persistence_file_path` is not configured" do + it 'aborts with a message explaining the config option must be set first' do + configuration.example_status_persistence_file_path = nil + world.announce_filters + expect(reporter).to have_received(:abort_with).with(/example_status_persistence_file_path/, 1) + end + end + + context "and `example_status_persistence_file_path` is configured" do + it 'does not abort' do + configuration.example_status_persistence_file_path = "foo.txt" + world.announce_filters + expect(reporter).not_to have_received(:abort_with) + end + end + end + + context "when --only-pending is not passed" do + before { expect(configuration.only_pending?).not_to eq true } + + context "and `example_status_persistence_file_path` is not configured" do + it 'does not abort' do + configuration.example_status_persistence_file_path = nil + world.announce_filters + expect(reporter).not_to have_received(:abort_with) + end + end + + context "and `example_status_persistence_file_path` is configured" do + it 'does not abort' do + configuration.example_status_persistence_file_path = "foo.txt" + world.announce_filters + expect(reporter).not_to have_received(:abort_with) + end + end + end + context "with no examples" do before { allow(world).to receive(:example_count) { 0 } }