Switching drop-in package removes overlapping files

[asottile]

The reason for this is both packages provide the same file. We install first and then uninstall after, removing the shared files from the new package.

This should reproduce:

set -ex

function dotest() {
    local req=$1
    echo "$req" > reqs.txt
    python venv_update.py venv reqs.txt
    ./venv/bin/python -c 'import MySQLdb'
}

# Demonstrate it works by itself
rm -rf venv
dotest 'mysql-python'
rm -rf venv
dotest 'mysqlclient'
rm -rf venv

# Now break it
dotest 'mysql-python'
dotest 'mysqlclient'

$ bash test.sh
+ rm -rf venv
+ dotest mysql-python
+ local req=mysql-python
+ echo mysql-python
+ python venv_update.py venv reqs.txt
> /usr/bin/python -m virtualenv venv
New python executable in /home/asottile/workspace/venv-update/venv/bin/python
Installing setuptools, pip, wheel...done.
> pip --version
pip 7.0.3 from /usr/local/lib/python2.7/dist-packages (python 2.7)
> venv/bin/python -m pip.__main__ install 'pip>=1.5.0,<6.0.0'
You are using pip version 7.1.0, however version 7.1.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
DEPRECATION: --download-cache has been deprecated and will be removed in the future. Pip now automatically uses and configures its cache.
Collecting pip<6.0.0,>=1.5.0
  Using cached pip-1.5.6-py2.py3-none-any.whl
Installing collected packages: pip
  Found existing installation: pip 7.1.0
    Uninstalling pip-7.1.0:
      Successfully uninstalled pip-7.1.0
Successfully installed pip-1.5.6
> venv/bin/python venv_update.py --stage2 venv reqs.txt
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse wheel==0.24.0
Requirement already up-to-date: wheel==0.24.0 in ./venv/lib/python2.7/site-packages
Cleaning up...
> pip wheel --wheel-dir=/home/asottile/.pip/wheelhouse wheel==0.24.0 --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --requirement=reqs.txt
Downloading/unpacking wheel==0.24.0
  File was already downloaded /home/asottile/.pip/wheelhouse/wheel-0.24.0-py2.py3-none-any.whl
Downloading/unpacking mysql-python (from -r reqs.txt (line 1))
  File was already downloaded /home/asottile/.pip/wheelhouse/MySQL_python-1.2.5-cp27-none-linux_x86_64.whl
Cleaning up...
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --no-index --requirement=reqs.txt
Ignoring indexes: https://pypi.python.org/simple/
Downloading/unpacking mysql-python (from -r reqs.txt (line 1))
Installing collected packages: mysql-python
Successfully installed mysql-python
Cleaning up...
+ ./venv/bin/python -c 'import MySQLdb'
+ rm -rf venv
+ dotest mysqlclient
+ local req=mysqlclient
+ echo mysqlclient
+ python venv_update.py venv reqs.txt
> /usr/bin/python -m virtualenv venv
New python executable in /home/asottile/workspace/venv-update/venv/bin/python
Installing setuptools, pip, wheel...done.
> pip --version
pip 7.0.3 from /usr/local/lib/python2.7/dist-packages (python 2.7)
> venv/bin/python -m pip.__main__ install 'pip>=1.5.0,<6.0.0'
You are using pip version 7.1.0, however version 7.1.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
DEPRECATION: --download-cache has been deprecated and will be removed in the future. Pip now automatically uses and configures its cache.
Collecting pip<6.0.0,>=1.5.0
  Using cached pip-1.5.6-py2.py3-none-any.whl
Installing collected packages: pip
  Found existing installation: pip 7.1.0
    Uninstalling pip-7.1.0:
      Successfully uninstalled pip-7.1.0
Successfully installed pip-1.5.6
> venv/bin/python venv_update.py --stage2 venv reqs.txt
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse wheel==0.24.0
Requirement already up-to-date: wheel==0.24.0 in ./venv/lib/python2.7/site-packages
Cleaning up...
> pip wheel --wheel-dir=/home/asottile/.pip/wheelhouse wheel==0.24.0 --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --requirement=reqs.txt
Downloading/unpacking wheel==0.24.0
  File was already downloaded /home/asottile/.pip/wheelhouse/wheel-0.24.0-py2.py3-none-any.whl
Downloading/unpacking mysqlclient (from -r reqs.txt (line 1))
  File was already downloaded /home/asottile/.pip/wheelhouse/mysqlclient-1.3.6-cp27-none-linux_x86_64.whl
Cleaning up...
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --no-index --requirement=reqs.txt
Ignoring indexes: https://pypi.python.org/simple/
Downloading/unpacking mysqlclient (from -r reqs.txt (line 1))
Installing collected packages: mysqlclient
Successfully installed mysqlclient
Cleaning up...
+ ./venv/bin/python -c 'import MySQLdb'
+ rm -rf venv
+ dotest mysql-python
+ local req=mysql-python
+ echo mysql-python
+ python venv_update.py venv reqs.txt
> /usr/bin/python -m virtualenv venv
New python executable in /home/asottile/workspace/venv-update/venv/bin/python
Installing setuptools, pip, wheel...done.
> pip --version
pip 7.0.3 from /usr/local/lib/python2.7/dist-packages (python 2.7)
> venv/bin/python -m pip.__main__ install 'pip>=1.5.0,<6.0.0'
You are using pip version 7.1.0, however version 7.1.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
DEPRECATION: --download-cache has been deprecated and will be removed in the future. Pip now automatically uses and configures its cache.
Collecting pip<6.0.0,>=1.5.0
  Using cached pip-1.5.6-py2.py3-none-any.whl
Installing collected packages: pip
  Found existing installation: pip 7.1.0
    Uninstalling pip-7.1.0:
      Successfully uninstalled pip-7.1.0
Successfully installed pip-1.5.6
> venv/bin/python venv_update.py --stage2 venv reqs.txt
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse wheel==0.24.0
Requirement already up-to-date: wheel==0.24.0 in ./venv/lib/python2.7/site-packages
Cleaning up...
> pip wheel --wheel-dir=/home/asottile/.pip/wheelhouse wheel==0.24.0 --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --requirement=reqs.txt
Downloading/unpacking wheel==0.24.0
  File was already downloaded /home/asottile/.pip/wheelhouse/wheel-0.24.0-py2.py3-none-any.whl
Downloading/unpacking mysql-python (from -r reqs.txt (line 1))
  File was already downloaded /home/asottile/.pip/wheelhouse/MySQL_python-1.2.5-cp27-none-linux_x86_64.whl
Cleaning up...
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --no-index --requirement=reqs.txt
Ignoring indexes: https://pypi.python.org/simple/
Downloading/unpacking mysql-python (from -r reqs.txt (line 1))
Installing collected packages: mysql-python
Successfully installed mysql-python
Cleaning up...
+ ./venv/bin/python -c 'import MySQLdb'
+ dotest mysqlclient
+ local req=mysqlclient
+ echo mysqlclient
+ python venv_update.py venv reqs.txt
Keeping virtualenv from previous run.
> pip --version
pip 7.0.3 from /usr/local/lib/python2.7/dist-packages (python 2.7)
> venv/bin/python -m pip.__main__ install 'pip>=1.5.0,<6.0.0'
Requirement already satisfied (use --upgrade to upgrade): pip>=1.5.0,<6.0.0 in ./venv/lib/python2.7/site-packages
Cleaning up...
> venv/bin/python venv_update.py --stage2 venv reqs.txt
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse wheel==0.24.0
Requirement already up-to-date: wheel==0.24.0 in ./venv/lib/python2.7/site-packages
Cleaning up...
> pip wheel --wheel-dir=/home/asottile/.pip/wheelhouse wheel==0.24.0 --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --requirement=reqs.txt
Downloading/unpacking wheel==0.24.0
  File was already downloaded /home/asottile/.pip/wheelhouse/wheel-0.24.0-py2.py3-none-any.whl
Downloading/unpacking mysqlclient (from -r reqs.txt (line 1))
  File was already downloaded /home/asottile/.pip/wheelhouse/mysqlclient-1.3.6-cp27-none-linux_x86_64.whl
Cleaning up...
> pip install --upgrade --use-wheel --download-cache=/home/asottile/.pip/cache --find-links=file:///home/asottile/.pip/wheelhouse --no-index --requirement=reqs.txt
Ignoring indexes: https://pypi.python.org/simple/
Downloading/unpacking mysqlclient (from -r reqs.txt (line 1))
Installing collected packages: mysqlclient
Successfully installed mysqlclient
Cleaning up...
> pip uninstall --yes mysql-python
Uninstalling MySQL-python:
  Successfully uninstalled MySQL-python
+ ./venv/bin/python -c 'import MySQLdb'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
ImportError: No module named MySQLdb

[bukzor]

New example: Removing a requirement for pip-faster and adding a requirement for venv-update at the same time results in deletion of the pip-faster script.

Fixing this would be very invasive surgery though.

[asottile]

Since I might try and fix it, here's an easier reproduction:

$ tail -n999 test.sh drop_in_*/*
==> test.sh <==
#!/usr/bin/env bash
set -euxo pipefail

rm -rf venv-test
virtualenv venv-test
venv-test/bin/pip install -e .

venv-test/bin/pip-faster install --prune ./drop_in_1
venv-test/bin/pip-faster install --prune ./drop_in_2
venv-test/bin/python -c 'import drop_in_mod'

==> drop_in_1/drop_in_mod.py <==

==> drop_in_1/setup.py <==
from setuptools import setup

setup(
    name='drop_in_1',
    py_modules=['drop_in_mod'],
)

==> drop_in_2/drop_in_mod.py <==

==> drop_in_2/setup.py <==
from setuptools import setup

setup(
    name='drop_in_2',
    py_modules=['drop_in_mod'],
)

[asottile]

This is where one would run the prune probably? https://github.com/pypa/pip/blob/094afefe2650625a9f243e3b1aec0345fbf5b9ba/src/pip/_internal/commands/install.py#L294