Allow missing .py files if .pyc is present

docs/changelog/1714.bugfix.rst

@@ -0,0 +1 @@
+Allow missing ``.py`` files if a compiled ``.pyc`` version is available - by :user:`tucked`.

docs/changelog/1714.doc.rst

@@ -0,0 +1,2 @@
+:ref:`supports <compatibility-requirements>` details now explicitly what Python installations we support
+- by :user:`gaborbernat`.

docs/installation.rst

@@ -89,8 +89,39 @@ virtualenv works with the following Python interpreter implementations:
 - `PyPy <https://pypy.org/>`_ 2.7 and 3.4+.
 
 This means virtualenv works on the latest patch version of each of these minor versions. Previous patch versions are
-supported on a best effort approach. virtualenv works on the following platforms:
+supported on a best effort approach.
 
-- Unix/Linux,
-- macOS,
-- Windows.
+CPython is shipped in multiple forms, and each OS repackages it, often applying some customization along the way.
+Therefore we cannot say universally that we support all platforms, but rather specify some we test against. In case
+of ones not specified here the support is unknown, though likely will work. If you find some cases please open a feature
+request on our issue tracker.
+
+Linux
+~~~~~
+- installations from `python.org <https://www.python.org/downloads/>`_
+- Ubuntu 16.04+ (both upstream and `deasnakes <https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa>`_ builds)
+- Fedora
+- RHEL and CentOS
+- OpenSuse
+- Arch Linux
+
+macOS
+~~~~~
+In case of macOs we support:
+- installations from `python.org <https://www.python.org/downloads/>`_
+- python versions installed via `brew <https://docs.brew.sh/Homebrew-and-Python>`_ (both older python2.7 and python3)
+- Python 3 part of XCode (Python framework - ``/Library/Frameworks/Python3.framework/``)
+- Python 2 part of the OS (``/System/Library/Frameworks/Python.framework/Versions/``)
+
+Windows
+~~~~~~~
+- Installations from `python.org <https://www.python.org/downloads/>`_
+- Windows Store Python - note only `version 3.8+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_ (``3.7``
+ was marked experimental and contains many bugs that would make it very hard for us to support it)
+
+Packaging variants
+~~~~~~~~~~~~~~~~~~
+- Normal variant (file structure as comes from `python.org <https://www.python.org/downloads/>`_).
+- We support CPython 2 system installations that do not contain the python files for the standard library if the
+ respective compiled files are present (e.g. only ``os.pyc``, not ``os.py``). This can be used by custom systems may
+ want to maximize available storage or obfuscate source code by removing ``.py`` files.

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -25,6 +25,10 @@ def sources(cls, interpreter):
  if host_include_marker.exists():
  yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
 
+ @classmethod
+ def needs_stdlib_py_module(cls):
+ return False
+
  @classmethod
  def host_include_marker(cls, interpreter):
  return Path(interpreter.system_include) / "Python.h"

src/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py

@@ -31,6 +31,10 @@ def sources(cls, interpreter):
  if host_include_marker.exists():
  yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
 
+ @classmethod
+ def needs_stdlib_py_module(cls):
+ return True
+
  @classmethod
  def host_include_marker(cls, interpreter):
  return Path(interpreter.system_include) / "PyPy.h"

src/virtualenv/create/via_global_ref/builtin/python2/python2.py

@@ -57,11 +57,27 @@ def sources(cls, interpreter):
  yield src
  # install files needed to run site.py
  for req in cls.modules():
- stdlib_path = interpreter.stdlib_path("{}.py".format(req))
- yield PathRefToDest(stdlib_path, dest=cls.to_stdlib)
- comp = stdlib_path.parent / "{}.pyc".format(req)
- if comp.exists():
- yield PathRefToDest(comp, dest=cls.to_stdlib)
+
+ # the compiled path is optional, but refer to it if exists
+ module_compiled_path = interpreter.stdlib_path("{}.pyc".format(req))
+ has_compile = module_compiled_path.exists()
+ if has_compile:
+ yield PathRefToDest(module_compiled_path, dest=cls.to_stdlib)
+
+ # stdlib module src may be missing if the interpreter allows it by falling back to the compiled
+ module_path = interpreter.stdlib_path("{}.py".format(req))
+ add_py_module = cls.needs_stdlib_py_module()
+ if add_py_module is False:
+ if module_path.exists(): # if present add it
+ add_py_module = True
+ else:
+ add_py_module = not has_compile # otherwise only add it if the pyc is not present
+ if add_py_module:
+ yield PathRefToDest(module_path, dest=cls.to_stdlib)
+
+ @classmethod
+ def needs_stdlib_py_module(cls):
+ raise NotImplementedError
 
  def to_stdlib(self, src):
  return self.stdlib / src.name

tests/unit/create/test_creator.py

@@ -21,7 +21,7 @@
 from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info
 from virtualenv.discovery.builtin import get_interpreter
 from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.info import IS_PYPY, IS_WIN, PY3, fs_is_case_sensitive, fs_supports_symlink
+from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive, fs_supports_symlink
 from virtualenv.pyenv_cfg import PyEnvCfg
 from virtualenv.run import cli_run, session_via_cli
 from virtualenv.util.path import Path
@@ -448,3 +448,36 @@ def _get_sys_path(flag=None):
  assert non_python_path == [i for i in base if i not in extra_as_python_path]
  else:
  assert base == extra_all
+
+
+@pytest.mark.skipif(
+ not (CURRENT.implementation == "CPython" and PY2),
+ reason="stdlib components without py files only possible on CPython2",
+)
+def test_pyc_only(tmp_path, mocker, session_app_data):
+ """Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)"""
+ interpreter = PythonInfo.from_exe(sys.executable, session_app_data)
+ host_pyc = interpreter.stdlib_path("os.pyc")
+ if not host_pyc.exists():
+ pytest.skip("missing system os.pyc at {}".format(host_pyc))
+ previous = interpreter.stdlib_path
+
+ def stdlib_path(name):
+ path = previous(name)
+ if name.endswith(".py"):
+
+ class _Path(type(path)):
+ @staticmethod
+ def exists():
+ return False
+
+ return _Path(path)
+ return path
+
+ mocker.patch.object(interpreter, "stdlib_path", side_effect=stdlib_path)
+
+ result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", ""])
+
+ assert not (result.creator.stdlib / "os.py").exists()
+ assert (result.creator.stdlib / "os.pyc").exists()
+ assert "os.pyc" in result.creator.debug["os"]