Regression in 2020.05.28 -- pipenv omits installation of dependencies in enclosing virtualenv if they are installed system-wide

[wpodgorski]

Issue description

Let's assume that we have some system-wide pip dependencies installed. In our scenario we are creating and activating new virtualenv which starts fresh, without any dependencies. Next we are installing all dependencies from the Pipfile.lock using pipenv install --ignore-pipfile --deploy --dev. Pipenv informs that it will use the virtualenv that has been activated in previous step:

Courtesy Notice: Pipenv found itself running within a virtual environment, so it will automatically use that environment, instead of creating its own for any project. You can set PIPENV_IGNORE_VIRTUALENVS=1 to force pipenv to ignore that environment and create its own instead. You can set PIPENV_VERBOSITY=-1 to suppress this warning.

Once the installation is done we can see that dependencies that were previously installed system-wide were omitted and not installed in the virtualenv.

Pipfile

[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true

[dev-packages]
pytest = "==5.4.1"
pytest-cov = "==2.8.1"

[packages]
boto3 = "==1.12.39"
requests = "==2.23.0"
zipstream-new = "==1.1.7"

[requires]
python_version = "3.7"

Pipfile.lock

{
    "_meta": {
        "hash": {
            "sha256": "20a3bc4432a7aac37224b2fbb32564689fe917920e9b537c40db47251159121c"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3.7"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "boto3": {
            "hashes": [
                "sha256:970bd7b332e73d7b51077ed36772c634811b38c81b0cc6ed0f910e50d7ebadf8",
                "sha256:cdd79a3a7bbe1f33a365f0acfcc75c4405b482b3eb9ce3f4e6b16c418e201ac3"
            ],
            "index": "pypi",
            "version": "==1.12.39"
        },
        "botocore": {
            "hashes": [
                "sha256:a474131ba7a7d700b91696a27e8cdcf1b473084addf92f90b269ebd8f5c3d3e0",
                "sha256:b805691b4dedcb2a252f52347479ff351429624a873f001b6a1c81aca03dccee"
            ],
            "version": "==1.15.49"
        },
        "certifi": {
            "hashes": [
                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
            ],
            "version": "==2020.4.5.1"
        },
        "chardet": {
            "hashes": [
                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
            ],
            "version": "==3.0.4"
        },
        "docutils": {
            "hashes": [
                "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
                "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827",
                "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"
            ],
            "version": "==0.15.2"
        },
        "idna": {
            "hashes": [
                "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb",
                "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"
            ],
            "version": "==2.9"
        },
        "jmespath": {
            "hashes": [
                "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9",
                "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"
            ],
            "version": "==0.10.0"
        },
        "python-dateutil": {
            "hashes": [
                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
            ],
            "version": "==2.8.1"
        },
        "requests": {
            "hashes": [
                "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
                "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"
            ],
            "index": "pypi",
            "version": "==2.23.0"
        },
        "s3transfer": {
            "hashes": [
                "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13",
                "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"
            ],
            "version": "==0.3.3"
        },
        "six": {
            "hashes": [
                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
            ],
            "version": "==1.15.0"
        },
        "urllib3": {
            "hashes": [
                "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
                "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
            ],
            "markers": "python_version != '3.4'",
            "version": "==1.25.9"
        },
        "zipstream-new": {
            "hashes": [
                "sha256:779d47c2c47934db610f8ce25ad5bd3bee33d805fe3ea1675f50ba0b01fb6497",
                "sha256:c5708462c5a12e227b3407e46489b5ffb516c38330983f1638d6cea83e653fb1"
            ],
            "index": "pypi",
            "version": "==1.1.7"
        }
    },
    "develop": {
        "attrs": {
            "hashes": [
                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
            ],
            "version": "==19.3.0"
        },
        "coverage": {
            "hashes": [
                "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a",
                "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355",
                "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65",
                "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7",
                "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9",
                "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1",
                "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0",
                "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55",
                "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c",
                "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6",
                "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef",
                "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019",
                "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e",
                "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0",
                "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf",
                "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24",
                "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2",
                "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c",
                "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4",
                "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0",
                "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd",
                "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04",
                "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e",
                "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730",
                "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2",
                "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768",
                "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796",
                "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7",
                "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a",
                "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489",
                "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052"
            ],
            "version": "==5.1"
        },
        "importlib-metadata": {
            "hashes": [
                "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
                "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
            ],
            "markers": "python_version < '3.8'",
            "version": "==1.6.0"
        },
        "more-itertools": {
            "hashes": [
                "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be",
                "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"
            ],
            "version": "==8.3.0"
        },
        "packaging": {
            "hashes": [
                "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8",
                "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"
            ],
            "version": "==20.4"
        },
        "pluggy": {
            "hashes": [
                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
            ],
            "version": "==0.13.1"
        },
        "py": {
            "hashes": [
                "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa",
                "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"
            ],
            "version": "==1.8.1"
        },
        "pyparsing": {
            "hashes": [
                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
            ],
            "version": "==2.4.7"
        },
        "pytest": {
            "hashes": [
                "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172",
                "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970"
            ],
            "index": "pypi",
            "version": "==5.4.1"
        },
        "pytest-cov": {
            "hashes": [
                "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b",
                "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"
            ],
            "index": "pypi",
            "version": "==2.8.1"
        },
        "six": {
            "hashes": [
                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
            ],
            "version": "==1.15.0"
        },
        "wcwidth": {
            "hashes": [
                "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1",
                "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"
            ],
            "version": "==0.1.9"
        },
        "zipp": {
            "hashes": [
                "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
                "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
            ],
            "version": "==3.1.0"
        }
    }
}

Expected result

All dependencies defined in Pipfile.lock should be installed in the virtualenv. Expected set of dependencies:

$> pip freeze

attrs==19.3.0
boto3==1.12.39
botocore==1.15.49
certifi==2020.4.5.1
chardet==3.0.4
coverage==5.1
docutils==0.15.2
idna==2.9
importlib-metadata==1.6.0
jmespath==0.10.0
more-itertools==8.3.0
packaging==20.4
pluggy==0.13.1
py==1.8.1
pyparsing==2.4.7
pytest==5.4.1
pytest-cov==2.8.1
python-dateutil==2.8.1
requests==2.23.0
s3transfer==0.3.3
six==1.15.0
urllib3==1.25.9
wcwidth==0.1.9
zipp==3.1.0
zipstream-new==1.1.7

Actual result

Only dependencies not being installed system-wide are installed in the virtualenv. Actual set of installed depedencies:

$> pip freeze

attrs==19.3.0
boto3==1.12.39
botocore==1.15.49
chardet==3.0.4
coverage==5.1
docutils==0.15.2
idna==2.9
jmespath==0.10.0
more-itertools==8.3.0
packaging==20.4
pluggy==0.13.1
py==1.8.1
pyparsing==2.4.7
pytest==5.4.1
pytest-cov==2.8.1
python-dateutil==2.8.1
requests==2.23.0
s3transfer==0.3.3
urllib3==1.25.9
wcwidth==0.1.9
zipstream-new==1.1.7

Steps to replicate

1. Add some system-wide dependencies

root@209c93f34a0a:~# pip freeze

appdirs==1.4.4
certifi==2020.4.5.1
distlib==0.3.0
filelock==3.0.12
importlib-metadata==1.6.0
pipenv==2020.5.28
six==1.15.0
virtualenv==20.0.21
virtualenv-clone==0.5.4
zipp==3.1.0

2. Go to Pipfile enclosing directory

root@209c93f34a0a:~# cd /project/

3. Create new virtualenv

root@209c93f34a0a:/project# virtualenv -p python venv

created virtual environment CPython3.7.7.final.0-64 in 2288ms
  creator CPython3Posix(dest=/project/venv, clear=False, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/home/node/.local/share/virtualenv/seed-app-data/v1.0.1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator

4. Activate virtualenv

root@209c93f34a0a:/project# source venv/bin/activate

5. Check if activated virtualenv is empty

(venv) root@209c93f34a0a:/project# pip freeze

6. Run pipenv install within the activated virtualenv

(venv) root@209c93f34a0a:/project# pipenv install --dev --deploy --ignore-pipfile

Courtesy Notice: Pipenv found itself running within a virtual environment, so it will automatically use that environment, instead of creating its own for any project. You can set PIPENV_IGNORE_VIRTUALENVS=1 to force pipenv to ignore that environment and create its own instead. You can set PIPENV_VERBOSITY=-1 to suppress this warning.
Installing dependencies from Pipfile.lock (59121c)…
  🐍   ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 21/21 — 00:00:23

7. Check what has been installed

(venv) root@209c93f34a0a:/project# pip freeze

attrs==19.3.0
boto3==1.12.39
botocore==1.15.49
chardet==3.0.4
coverage==5.1
docutils==0.15.2
idna==2.9
jmespath==0.10.0
more-itertools==8.3.0
packaging==20.4
pluggy==0.13.1
py==1.8.1
pyparsing==2.4.7
pytest==5.4.1
pytest-cov==2.8.1
python-dateutil==2.8.1
requests==2.23.0
s3transfer==0.3.3
urllib3==1.25.9
wcwidth==0.1.9
zipstream-new==1.1.7

---

$ pipenv --support

Pipenv version: '2020.5.28'

Pipenv location: '/usr/local/lib/python3.7/site-packages/pipenv'

Python location: '/usr/local/bin/python'

Python installations found:

• 3.7.7: /usr/local/bin/python3.7m
• 3.7.7: /usr/local/bin/python3
• 3.7.7: /usr/local/bin/python3.7
• 3.7.3: /usr/bin/python3.7m
• 3.7.3: /usr/bin/python3
• 3.7.3: /usr/bin/python3.7
• 2.7.16: /usr/bin/python2
• 2.7.16: /usr/bin/python2.7

PEP 508 Information:

{'implementation_name': 'cpython',
 'implementation_version': '3.7.7',
 'os_name': 'posix',
 'platform_machine': 'x86_64',
 'platform_python_implementation': 'CPython',
 'platform_release': '4.19.76-linuxkit',
 'platform_system': 'Linux',
 'platform_version': '#1 SMP Fri Apr 3 15:53:26 UTC 2020',
 'python_full_version': '3.7.7',
 'python_version': '3.7',
 'sys_platform': 'linux'}

System environment variables:

• HOSTNAME
• PYTHON_VERSION
• PWD
• HOME
• LANG
• GPG_KEY
• TERM
• PROJECT_PATH
• SHLVL
• PYTHON_PIP_VERSION
• PYTHON_GET_PIP_SHA256
• PYTHON_GET_PIP_URL
• PATH
• _
• PIP_DISABLE_PIP_VERSION_CHECK
• PYTHONDONTWRITEBYTECODE
• PIP_SHIMS_BASE_MODULE
• PIP_PYTHON_PATH
• PYTHONFINDER_IGNORE_UNSUPPORTED

Pipenv–specific environment variables:

Debug–specific environment variables:

• PATH: /usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
• LANG: C.UTF-8
• PWD: /home/node

---

[jacobsee]

+1
A number of builds I have are failing on the latest Pipenv due to this issue.

[techalchemy]

Can someone comment on whether this occurs for non-root users?

[frostming]

I can reproduce the issue with the exact steps from the OP.