From c1f62e82fe275471085e1d914caa8b73cdefc091 Mon Sep 17 00:00:00 2001 From: Jurre Stender Date: Wed, 22 Dec 2021 17:20:30 +0100 Subject: [PATCH] Python: Quarantine slow tests into their own CI run These tests have not been wearing masks, and to avoid them infecting the other tests we need to quarantine them. Jokes aside, splitting these slow tests off into their own CI run should reduce the overall time we have to wait on CI to pass, and it gives us some indication on which tests we might want to try to speed up going forward. So far I've split off the ~10 slowest tests, which according to rspec take 40% of the total time to run these tests! We can move more things into the slow suite, but I'd like to avoid having the majority of the tests live there, because then we'd still end up with one very slow suite. --- .github/workflows/ci.yml | 1 + .../pip_compile_file_updater_spec.rb | 6 +++--- .../file_updater/pipfile_file_updater_spec.rb | 4 ++-- .../pip_compile_version_resolver_spec.rb | 6 +++--- .../pipenv_version_resolver_spec.rb | 2 +- .../poetry_version_resolver_spec.rb | 2 +- python/spec/spec_helper.rb | 18 ++++++++++++++++++ 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ee78e1356..5ef883c320 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: - { path: nuget, name: nuget } - { path: omnibus, name: omnibus } - { path: python, name: python } + - { path: python, name: python_slow } - { path: terraform, name: terraform } env: BASE_IMAGE: ubuntu:18.04 diff --git a/python/spec/dependabot/python/file_updater/pip_compile_file_updater_spec.rb b/python/spec/dependabot/python/file_updater/pip_compile_file_updater_spec.rb index f65c16340d..a500bb0000 100644 --- a/python/spec/dependabot/python/file_updater/pip_compile_file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater/pip_compile_file_updater_spec.rb @@ -223,7 +223,7 @@ let(:generated_fixture_name) { "pip_compile_imports_setup.txt" } let(:setup_fixture_name) { "small.py" } - it "updates the requirements.txt" do + it "updates the requirements.txt", :slow do expect(updated_files.count).to eq(1) expect(updated_files.first.content).to include("attrs==18.1.0") expect(updated_files.first.content). @@ -235,7 +235,7 @@ expect(updated_files.first.content).to_not include("--hash=sha") end - context "that needs sanitizing" do + context "that needs sanitizing", :slow do let(:setup_fixture_name) { "small_needs_sanitizing.py" } it "updates the requirements.txt" do expect(updated_files.count).to eq(1) @@ -497,7 +497,7 @@ let(:dependency_requirements) { [] } let(:dependency_previous_requirements) { [] } - it "raises an error indicating the dependencies are not resolvable" do + it "raises an error indicating the dependencies are not resolvable", :slow do expect { updated_files }.to raise_error(Dependabot::DependencyFileNotResolvable) do |err| expect(err.message).to include( "There are incompatible versions in the resolved dependencies:\n pyyaml==5.4" diff --git a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb index 8a15699a23..5125f789cb 100644 --- a/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb +++ b/python/spec/dependabot/python/file_updater/pipfile_file_updater_spec.rb @@ -133,7 +133,7 @@ ) end - it "updates both files correctly" do + it "updates both files correctly", :slow do expect(updated_files.map(&:name)).to eq(%w(Pipfile Pipfile.lock)) updated_lockfile = updated_files.find { |f| f.name == "Pipfile.lock" } @@ -310,7 +310,7 @@ let(:lockfile_fixture_name) { "git_source_no_ref.lock" } context "when updating the non-git dependency" do - it "doesn't update the git dependency" do + it "doesn't update the git dependency", :slow do expect(json_lockfile["default"]["requests"]["version"]). to eq("==2.18.4") expect(json_lockfile["default"]["pythonfinder"]). diff --git a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb index 02824a3578..0f87758aec 100644 --- a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb @@ -269,7 +269,7 @@ }] end - it "raises a helpful error" do + it "raises a helpful error", :slow do expect { subject }. to raise_error(Dependabot::DependencyFileNotResolvable) do |error| expect(error.message). @@ -337,7 +337,7 @@ it { is_expected.to be >= Gem::Version.new("40.6.2") } end - context "with an import of the setup.py" do + context "with an import of the setup.py", :slow do let(:dependency_files) do [manifest_file, generated_file, setup_file, pyproject] end @@ -375,7 +375,7 @@ end end - context "with native dependencies that are not pre-built" do + context "with native dependencies that are not pre-built", :slow do let(:manifest_fixture_name) { "native_dependencies.in" } let(:generated_fixture_name) { "pip_compile_native_dependencies.txt" } let(:dependency_name) { "cryptography" } diff --git a/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb index b69ef5503c..9c9bbd3325 100644 --- a/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/pipenv_version_resolver_spec.rb @@ -247,7 +247,7 @@ end end - context "for a resolution that has caused trouble in the past" do + context "for a resolution that has caused trouble in the past", :slow do let(:dependency_files) { [pipfile] } let(:pipfile_fixture_name) { "problematic_resolution" } let(:dependency_name) { "twilio" } diff --git a/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb index 0d88dbf5f1..2dd2130e10 100644 --- a/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/poetry_version_resolver_spec.rb @@ -204,7 +204,7 @@ it { is_expected.to eq(Gem::Version.new("2.15.1")) } end - context "resolvable only if git references are preserved" do + context "resolvable only if git references are preserved", :slow do let(:pyproject_fixture_name) { "git_conflict.toml" } let(:lockfile_fixture_name) { "git_conflict.lock" } let(:dependency_name) { "django-widget-tweaks" } diff --git a/python/spec/spec_helper.rb b/python/spec/spec_helper.rb index 6dcb854b37..46feec66c6 100644 --- a/python/spec/spec_helper.rb +++ b/python/spec/spec_helper.rb @@ -9,3 +9,21 @@ def require_common_spec(path) end require "#{common_dir}/spec/spec_helper.rb" + +module SlowTestHelper + def self.slow_tests? + ENV["SUITE_NAME"] == "python_slow" + end +end + +RSpec.configure do |config| + config.around do |example| + if SlowTestHelper.slow_tests? && example.metadata[:slow] + example.run + elsif !SlowTestHelper.slow_tests? && !example.metadata[:slow] + example.run + else + example.skip + end + end +end