Flaky tests

[atugushev]

test_bad_setup_file:

• first appearance on d8e272f
• failed on 1c67754
• passed on 5b3fa3c

Details

def test_bad_setup_file(runner):
        with open("setup.py", "w") as package:
            package.write("BAD SYNTAX")
    
        out = runner.invoke(cli, [])
    
>       assert out.exit_code == 2
E       AssertionError: assert 1 == 2
E        +  where 1 = <Result CalledProcessError(1, ['/private/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/build-env-gdwusm65/bin/python...p517', '--no-warn-script-location', '-r', '/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/build-reqs-ynj6e9y3.txt'])>.exit_code

test_direct_reference_with_extras:

• failed
• passed on a92a6af
• I've noticed test_direct_reference_with_extras doesn't have @pytest.mark.network marker.

✅ sync tests - addressed in #1761

• test_sync_install_temporary_requirement_file
• test_sync_up_to_date
• test_sync_verbose
• test_sync_uninstall_pip_command
• https://github.com/jazzband/pip-tools/actions/runs/3515128716/jobs/5890003279

Details

2022-11-21T14:12:58.8111071Z �[31m�[1m_________________ test_sync_install_temporary_requirement_file _________________�[0m
2022-11-21T14:12:58.8111819Z [gw1] linux -- Python 3.7.15 /home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python
2022-11-21T14:12:58.8112434Z 
2022-11-21T14:12:58.8112749Z from_line = <function install_req_from_line at 0x7f87d86bf4d0>
2022-11-21T14:12:58.8113460Z from_editable = <function install_req_from_editable at 0x7f87d86bf290>
2022-11-21T14:12:58.8114104Z mocked_tmp_req_file = <MagicMock name='NamedTemporaryFile()' id='140221262482128'>
2022-11-21T14:12:58.8114907Z 
2022-11-21T14:12:58.8115409Z     def test_sync_install_temporary_requirement_file(
2022-11-21T14:12:58.8116116Z         from_line, from_editable, mocked_tmp_req_file
2022-11-21T14:12:58.8116515Z     ):
2022-11-21T14:12:58.8118554Z         with mock.patch("piptools.sync.run") as run:
2022-11-21T14:12:58.8119006Z             to_install = {from_line("django==1.8")}
2022-11-21T14:12:58.8119727Z             sync(to_install, set())
2022-11-21T14:12:58.8120142Z             run.assert_called_once_with(
2022-11-21T14:12:58.8121000Z                 [sys.executable, "-m", "pip", "install", "-r", mocked_tmp_req_file.name],
2022-11-21T14:12:58.8121460Z >               check=True,
2022-11-21T14:12:58.8122018Z             )
2022-11-21T14:12:58.8122295Z 
2022-11-21T14:12:58.8122829Z �[1m�[31mtests/test_sync.py�[0m:275: 
2022-11-21T14:12:58.8123244Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2022-11-21T14:12:58.8124129Z �[1m�[31m/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/unittest/mock.py�[0m:889: in assert_called_once_with
2022-11-21T14:12:58.8124666Z     return self.assert_called_with(*args, **kwargs)
2022-11-21T14:12:58.8125306Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2022-11-21T14:12:58.8125613Z 
2022-11-21T14:12:58.8126178Z _mock_self = <MagicMock name='run' id='140221260076624'>
2022-11-21T14:12:58.8126941Z args = (['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'install', '-r', 'requirements.txt'],)
2022-11-21T14:12:58.8127727Z kwargs = {'check': True}
2022-11-21T14:12:58.8128466Z expected = ((['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'install', '-r', 'requirements.txt'],), {'check': True})
2022-11-21T14:12:58.8129349Z _error_message = <function NonCallableMock.assert_called_with.<locals>._error_message at 0x7f87ce689710>
2022-11-21T14:12:58.8130190Z actual = call(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'install', '-r', 'requirements.txt', '-q'], check=True)
2022-11-21T14:12:58.8131102Z cause = None
2022-11-21T14:12:58.8131387Z 
2022-11-21T14:12:58.8135362Z     def assert_called_with(_mock_self, *args, **kwargs):
2022-11-21T14:12:58.8135953Z         """assert that the mock was called with the specified arguments.
2022-11-21T14:12:58.8136602Z     
2022-11-21T14:12:58.8137051Z         Raises an AssertionError if the args and keyword args passed in are
2022-11-21T14:12:58.8137737Z         different to the last call to the mock."""
2022-11-21T14:12:58.8138150Z         self = _mock_self
2022-11-21T14:12:58.8138740Z         if self.call_args is None:
2022-11-21T14:12:58.8139180Z             expected = self._format_mock_call_signature(args, kwargs)
2022-11-21T14:12:58.8140073Z             raise AssertionError('Expected call: %s\nNot called' % (expected,))
2022-11-21T14:12:58.8140512Z     
2022-11-21T14:12:58.8142017Z         def _error_message():
2022-11-21T14:12:58.8142552Z             msg = self._format_mock_failure_message(args, kwargs)
2022-11-21T14:12:58.8143184Z             return msg
2022-11-21T14:12:58.8143598Z         expected = self._call_matcher((args, kwargs))
2022-11-21T14:12:58.8144254Z         actual = self._call_matcher(self.call_args)
2022-11-21T14:12:58.8144681Z         if expected != actual:
2022-11-21T14:12:58.8145338Z             cause = expected if isinstance(expected, Exception) else None
2022-11-21T14:12:58.8145841Z >           raise AssertionError(_error_message()) from cause
2022-11-21T14:12:58.8146948Z �[1m�[31mE           AssertionError: Expected call: run(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'install', '-r', 'requirements.txt'], check=True)�[0m
2022-11-21T14:12:58.8147958Z �[1m�[31mE           Actual call: run(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'install', '-r', 'requirements.txt', '-q'], check=True)�[0m
2022-11-21T14:12:58.8148640Z 
2022-11-21T14:12:58.8149374Z �[1m�[31m/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/unittest/mock.py�[0m:878: AssertionError
2022-11-21T14:12:58.8150269Z �[31m�[1m_____________________________ test_sync_up_to_date _____________________________�[0m
2022-11-21T14:12:58.8150938Z [gw1] linux -- Python 3.7.15 /home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python
2022-11-21T14:12:58.8151533Z 
2022-11-21T14:12:58.8151873Z capsys = <_pytest.capture.CaptureFixture object at 0x7f87ce7de910>
2022-11-21T14:12:58.8152573Z runner = <click.testing.CliRunner object at 0x7f87ce5f6310>
2022-11-21T14:12:58.8152920Z 
2022-11-21T14:12:58.8153390Z     def test_sync_up_to_date(capsys, runner):
2022-11-21T14:12:58.8153785Z         """
2022-11-21T14:12:58.8154471Z         Everything up-to-date should be printed.
2022-11-21T14:12:58.8154882Z         """
2022-11-21T14:12:58.8155437Z         sync(set(), set())
2022-11-21T14:12:58.8155842Z         captured = capsys.readouterr()
2022-11-21T14:12:58.8156603Z >       assert captured.out.splitlines() == ["Everything up-to-date"]
2022-11-21T14:12:58.8157733Z �[1m�[31mE       AssertionError: assert [] == ['Everything up-to-date']�[0m
2022-11-21T14:12:58.8158350Z �[1m�[31mE         Right contains one more item: 'Everything up-to-date'�[0m
2022-11-21T14:12:58.8159865Z �[1m�[31mE         Full diff:�[0m
2022-11-21T14:12:58.8160204Z �[1m�[31mE         - ['Everything up-to-date']�[0m
2022-11-21T14:12:58.8160503Z �[1m�[31mE         + []�[0m
2022-11-21T14:12:58.8160639Z 
2022-11-21T14:12:58.8160932Z �[1m�[31m/home/runner/work/pip-tools/pip-tools/tests/test_sync.py�[0m:377: AssertionError
2022-11-21T14:12:58.8161395Z �[31m�[1m______________________________ test_sync_verbose _______________________________�[0m
2022-11-21T14:12:58.8161894Z [gw1] linux -- Python 3.7.15 /home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python
2022-11-21T14:12:58.8162134Z 
2022-11-21T14:12:58.8162317Z run = <MagicMock name='run' id='140221259670224'>
2022-11-21T14:12:58.8162779Z from_line = <function install_req_from_line at 0x7f87d86bf4d0>
2022-11-21T14:12:58.8162964Z 
2022-11-21T14:12:58.8163093Z     @mock.patch("piptools.sync.run")
2022-11-21T14:12:58.8163374Z     def test_sync_verbose(run, from_line):
2022-11-21T14:12:58.8163604Z         """
2022-11-21T14:12:58.8163942Z         The -q option has to be passed to every pip calls.
2022-11-21T14:12:58.8164185Z         """
2022-11-21T14:12:58.8164444Z         sync({from_line("django==1.8")}, {from_line("click==4.0")})
2022-11-21T14:12:58.8164726Z         assert run.call_count == 2
2022-11-21T14:12:58.8164979Z         for call in run.call_args_list:
2022-11-21T14:12:58.8165226Z             run_args = call[0][0]
2022-11-21T14:12:58.8165513Z >           assert "-q" not in run_args
2022-11-21T14:12:58.8166127Z �[1m�[31mE           AssertionError: assert '-q' not in ['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', '-q', ...]�[0m
2022-11-21T14:12:58.8166434Z 
2022-11-21T14:12:58.8166638Z �[1m�[31mtests/test_sync.py�[0m:390: AssertionError
2022-11-21T14:12:58.8167066Z �[31m�[1m_______________________ test_sync_uninstall_pip_command ________________________�[0m
2022-11-21T14:12:58.8167568Z [gw1] linux -- Python 3.7.15 /home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python
2022-11-21T14:12:58.8167809Z 
2022-11-21T14:12:58.8168008Z run = <MagicMock name='run' id='140221259671504'>
2022-11-21T14:12:58.8168171Z 
2022-11-21T14:12:58.8168294Z     @mock.patch("piptools.sync.run")
2022-11-21T14:12:58.8168571Z     def test_sync_uninstall_pip_command(run):
2022-11-21T14:12:58.8168866Z         to_uninstall = ["six", "django", "pytz", "click"]
2022-11-21T14:12:58.8169104Z     
2022-11-21T14:12:58.8169315Z         sync(set(), to_uninstall)
2022-11-21T14:12:58.8169561Z         run.assert_called_once_with(
2022-11-21T14:12:58.8169972Z             [sys.executable, "-m", "pip", "uninstall", "-y", *sorted(to_uninstall)],
2022-11-21T14:12:58.8170246Z >           check=True,
2022-11-21T14:12:58.8170455Z         )
2022-11-21T14:12:58.8170572Z 
2022-11-21T14:12:58.8170739Z �[1m�[31mtests/test_sync.py�[0m:480: 
2022-11-21T14:12:58.8171081Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2022-11-21T14:12:58.8171580Z �[1m�[31m/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/unittest/mock.py�[0m:889: in assert_called_once_with
2022-11-21T14:12:58.8171949Z     return self.assert_called_with(*args, **kwargs)
2022-11-21T14:12:58.8172223Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2022-11-21T14:12:58.8172377Z 
2022-11-21T14:12:58.8172588Z _mock_self = <MagicMock name='run' id='140221259671504'>
2022-11-21T14:12:58.8173139Z args = (['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', 'click', ...],)
2022-11-21T14:12:58.8173550Z kwargs = {'check': True}
2022-11-21T14:12:58.8174106Z expected = ((['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', 'click', ...],), {'check': True})
2022-11-21T14:12:58.8174630Z _error_message = <function NonCallableMock.assert_called_with.<locals>._error_message at 0x7f87ce6890e0>
2022-11-21T14:12:58.8175324Z actual = call(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', '-q', 'click', 'django', 'pytz', 'six'], check=True)
2022-11-21T14:12:58.8175704Z cause = None
2022-11-21T14:12:58.8175836Z 
2022-11-21T14:12:58.8175977Z     def assert_called_with(_mock_self, *args, **kwargs):
2022-11-21T14:12:58.8176313Z         """assert that the mock was called with the specified arguments.
2022-11-21T14:12:58.8176577Z     
2022-11-21T14:12:58.8176855Z         Raises an AssertionError if the args and keyword args passed in are
2022-11-21T14:12:58.8177188Z         different to the last call to the mock."""
2022-11-21T14:12:58.8177429Z         self = _mock_self
2022-11-21T14:12:58.8177663Z         if self.call_args is None:
2022-11-21T14:12:58.8177962Z             expected = self._format_mock_call_signature(args, kwargs)
2022-11-21T14:12:58.8178485Z             raise AssertionError('Expected call: %s\nNot called' % (expected,))
2022-11-21T14:12:58.8178763Z     
2022-11-21T14:12:58.8178957Z         def _error_message():
2022-11-21T14:12:58.8179241Z             msg = self._format_mock_failure_message(args, kwargs)
2022-11-21T14:12:58.8179501Z             return msg
2022-11-21T14:12:58.8179802Z         expected = self._call_matcher((args, kwargs))
2022-11-21T14:12:58.8180096Z         actual = self._call_matcher(self.call_args)
2022-11-21T14:12:58.8180356Z         if expected != actual:
2022-11-21T14:12:58.8180644Z             cause = expected if isinstance(expected, Exception) else None
2022-11-21T14:12:58.8180981Z >           raise AssertionError(_error_message()) from cause
2022-11-21T14:12:58.8181754Z �[1m�[31mE           AssertionError: Expected call: run(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', 'click', 'django', 'pytz', 'six'], check=True)�[0m
2022-11-21T14:12:58.8182593Z �[1m�[31mE           Actual call: run(['/home/runner/work/pip-tools/pip-tools/.tox/pipmain-coverage/bin/python', '-m', 'pip', 'uninstall', '-y', '-q', 'click', 'django', 'pytz', 'six'], check=True)�[0m
2022-11-21T14:12:58.8182928Z 
2022-11-21T14:12:58.8183248Z �[1m�[31m/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/unittest/mock.py�[0m:878: AssertionError