From 4b4a24807d71a32674ffa840f6713c0197e1332e Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Wed, 5 Jul 2023 16:45:10 -0300 Subject: [PATCH 1/2] Refactor the project's install directory name. --- pyempaq/unpacker.py | 29 ++++++++++++++++++++++++++--- requirements-dev.txt | 1 + tests/test_unpacker.py | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pyempaq/unpacker.py b/pyempaq/unpacker.py index a8d5a41..3321fde 100644 --- a/pyempaq/unpacker.py +++ b/pyempaq/unpacker.py @@ -4,6 +4,8 @@ """Unpacking functionality..""" +import hashlib +import importlib import json import logging import os @@ -12,7 +14,6 @@ import shutil import subprocess import sys -import time import venv import zipfile from types import ModuleType @@ -155,6 +156,29 @@ def restrictions_ok(version: ModuleType, restrictions: Dict[str, Any]) -> bool: return True +def build_project_install_dir(zip_path: pathlib.Path, metadata: Dict[str, str]): + """Build the name of the directory where everything will be extracted.""" + project_name = metadata["project_name"] + + # get the first part of the hash of the file + hasher = hashlib.sha256() + with open(zip_path, "rb") as fh: + while True: + data = fh.read(65536) + hasher.update(data) + if not data: + break + file_hash_partial = hasher.hexdigest()[:20] + + # Python details + py_impl = platform.python_implementation().lower() + py_version = ".".join(platform.python_version_tuple()[:2]) + py_magic = importlib.util.MAGIC_NUMBER.strip().decode("ascii") + + name = f"{project_name}-{file_hash_partial}-{py_impl}.{py_version}.{py_magic}" + return name + + def run(): """Run the unpacker.""" log("PyEmpaq start") @@ -179,8 +203,7 @@ def run(): log("Temp base dir: %r", str(pyempaq_dir)) # create a temp dir and extract the project there - timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime(pyempaq_filepath.stat().st_ctime)) - project_dir = pyempaq_dir / "{}-{}".format(metadata["project_name"], timestamp) + project_dir = pyempaq_dir / build_project_install_dir(pyempaq_filepath, metadata) original_project_dir = project_dir / "orig" venv_requirements = [original_project_dir / fname for fname in metadata["requirement_files"]] setup_project_directory(zf, project_dir, venv_requirements) diff --git a/requirements-dev.txt b/requirements-dev.txt index 325e01e..81cc17e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ logassert packaging pydocstyle pytest +pytest-mock pytest-subprocess diff --git a/tests/test_unpacker.py b/tests/test_unpacker.py index f0b4e37..cbf45b4 100644 --- a/tests/test_unpacker.py +++ b/tests/test_unpacker.py @@ -4,6 +4,8 @@ """Unpacker tests.""" +import hashlib +import importlib import os import platform import zipfile @@ -18,6 +20,7 @@ from pyempaq.unpacker import ( PROJECT_VENV_DIR, build_command, + build_project_install_dir, restrictions_ok, run_command, setup_project_directory, @@ -279,3 +282,24 @@ def test_enforcerestrictions_pythonversion_good_comparison(logs): """Enforce minimum python version using a proper comparison, not strings.""" ok = restrictions_ok(version, {"minimum_python_version": "3.0009"}) assert ok is True + + +# --- tests for the project install dir name + + +def test_installdirname_complete(mocker, tmp_path): + """Check the name is properly built.""" + mocker.patch("platform.python_implementation", return_value="PyPy") + mocker.patch("platform.python_version_tuple", return_value=("3", "18", "7alpha")) + + zip_path = tmp_path / "somestuff.zip" + content = b"some content to be hashed" + zip_path.write_bytes(content) + content_hash = hashlib.sha256(content).hexdigest() + + fake_metadata = {"foo": "bar", "project_name": "testproj"} + + dirname = build_project_install_dir(zip_path, fake_metadata) + + magic = importlib.util.MAGIC_NUMBER.strip().decode("ascii") # scared to patch this one + assert dirname == f"testproj-{content_hash[:20]}-pypy.3.18.{magic}" From b26b97d010a85af2de430f0fbef7d06c9ba65150 Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Fri, 7 Jul 2023 22:45:19 -0300 Subject: [PATCH 2/2] Use magic number's hex. --- pyempaq/unpacker.py | 7 +++++-- tests/test_unpacker.py | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pyempaq/unpacker.py b/pyempaq/unpacker.py index 3321fde..750d298 100644 --- a/pyempaq/unpacker.py +++ b/pyempaq/unpacker.py @@ -29,6 +29,10 @@ # the file name to flag that the project setup completed successfully COMPLETE_FLAG_FILE = "complete.flag" +# the real magic number is a byte sequence with "\r\n" at the end; it's built here +# so it's easily patchable by tests +MAGIC_NUMBER = importlib.util.MAGIC_NUMBER[:-2].hex() + # setup logging logger = logging.getLogger() handler = logging.StreamHandler() @@ -173,9 +177,8 @@ def build_project_install_dir(zip_path: pathlib.Path, metadata: Dict[str, str]): # Python details py_impl = platform.python_implementation().lower() py_version = ".".join(platform.python_version_tuple()[:2]) - py_magic = importlib.util.MAGIC_NUMBER.strip().decode("ascii") - name = f"{project_name}-{file_hash_partial}-{py_impl}.{py_version}.{py_magic}" + name = f"{project_name}-{file_hash_partial}-{py_impl}.{py_version}.{MAGIC_NUMBER}" return name diff --git a/tests/test_unpacker.py b/tests/test_unpacker.py index cbf45b4..8a26c0e 100644 --- a/tests/test_unpacker.py +++ b/tests/test_unpacker.py @@ -5,7 +5,6 @@ """Unpacker tests.""" import hashlib -import importlib import os import platform import zipfile @@ -291,6 +290,7 @@ def test_installdirname_complete(mocker, tmp_path): """Check the name is properly built.""" mocker.patch("platform.python_implementation", return_value="PyPy") mocker.patch("platform.python_version_tuple", return_value=("3", "18", "7alpha")) + mocker.patch("pyempaq.unpacker.MAGIC_NUMBER", "xyz") zip_path = tmp_path / "somestuff.zip" content = b"some content to be hashed" @@ -301,5 +301,4 @@ def test_installdirname_complete(mocker, tmp_path): dirname = build_project_install_dir(zip_path, fake_metadata) - magic = importlib.util.MAGIC_NUMBER.strip().decode("ascii") # scared to patch this one - assert dirname == f"testproj-{content_hash[:20]}-pypy.3.18.{magic}" + assert dirname == f"testproj-{content_hash[:20]}-pypy.3.18.xyz"