Prepare support for local PyPI cross package dependencies (#71)

* build: tox.ini - define distshare, so that builds are kept

* build: tox.ini - define distshare as dist/tox to allow isolated clean

* build: make.py: improve clean, create local index for PyPi packages

make.py

@@ -10,7 +10,8 @@
 import shutil
 import subprocess
 import sys
-from typing import List
+from glob import glob
+from typing import Any, Dict, List
 
 import toml
 
@@ -36,16 +37,47 @@
 FACET_PATH_ENV = "FACET_PATH"
 FACET_BUILD_PKG_VERSION_ENV = "FACET_BUILD_{project}_VERSION"
 CONDA_BUILD_PATH_ENV = "CONDA_BLD_PATH"
-BUILD_PATH_SUFFIX = os.path.join("dist", "conda")
+CONDA_BUILD_PATH_SUFFIX = os.path.join("dist", "conda")
+TOX_BUILD_PATH_SUFFIX = os.path.join("dist", "tox")
 
 
-def make_build_path(project: str) -> str:
+def make_build_path(project: str, build_system: str) -> str:
     """
-    Return the target build path for Conda-build.
+    Return the target build path for Conda or Tox build.
     """
-    return os.path.abspath(
-        os.path.join(os.environ[FACET_PATH_ENV], project, BUILD_PATH_SUFFIX)
-    )
+    if build_system == B_CONDA:
+        return os.path.abspath(
+            os.path.join(os.environ[FACET_PATH_ENV], project, CONDA_BUILD_PATH_SUFFIX)
+        )
+    elif build_system == B_TOX:
+        return os.path.abspath(
+            os.path.join(os.environ[FACET_PATH_ENV], project, TOX_BUILD_PATH_SUFFIX)
+        )
+
+
+def make_local_pypi_index_path(project: str) -> str:
+    """
+    Return the path where the local PyPi index for
+    the given project should be placed.
+    """
+    return os.path.join(make_build_path(project, B_TOX), "simple")
+
+
+def get_pyproject_toml(project: str) -> Dict[str, Any]:
+    """
+    Retrieve a parsed Dict for a given project's pyproject.toml.
+    """
+    pyproject_toml_path = os.path.join(FACET_PATH, project, "pyproject.toml")
+    with open(pyproject_toml_path, "rt") as f:
+        return toml.load(f)
+
+
+def get_package_dist_name(project: str) -> str:
+    """
+    Retrieves from pyproject.toml for a project the appropriate
+    dist-name. E.g. "gamma-pytools" for project "pytools".
+    """
+    return get_pyproject_toml(project)["tool"]["flit"]["metadata"]["dist-name"]
 
 
 def get_package_version(project: str) -> str:
@@ -77,9 +109,7 @@ def expose_deps(project: str, build_system: str, dependency_type: str) -> None:
     """
     Expose package dependencies for builds as environment variables.
     """
-    pyproject_toml_path = os.path.join(FACET_PATH, project, "pyproject.toml")
-    with open(pyproject_toml_path, "rt") as f:
-        pyproject_toml = toml.load(f)
+    pyproject_toml = get_pyproject_toml(project)
 
     def adapt_version_syntax(version: str) -> str:
         if build_system == B_CONDA or (">" in version or "<" in version):
@@ -117,12 +147,33 @@ def adapt_version_syntax(version: str) -> str:
         os.environ[env_var_key] = env_var_value
 
 
+def clean(project: str, build_system: str) -> None:
+    """
+    Cleans the dist folder for the given project and build system.
+    """
+    build_path = make_build_path(project, build_system)
+    if build_system == B_CONDA:
+        # purge pre-existing build directories
+        package_dist_name = get_package_dist_name(project)
+        for obsolete_folder in glob(os.path.join(build_path, f"{package_dist_name}_*")):
+            print(f"Clean: Removing obsolete conda-build folder at: {obsolete_folder}")
+            shutil.rmtree(obsolete_folder, ignore_errors=True)
+
+        # remove broken packages
+        shutil.rmtree(os.path.join(build_path, "broken"), ignore_errors=True)
+
+    elif build_system == B_TOX:
+        # nothing to do – .tar.gz of same version will simply be replaced and
+        # .tox is useful to keep
+        pass
+
+
 def set_up(project: str, build_system: str, dependency_type: str) -> None:
     """
     Set up for a build – set FACET_PATH (parent folder of all projects) and clean.
     """
     os.environ[FACET_PATH_ENV] = FACET_PATH
-    shutil.rmtree(make_build_path(project), ignore_errors=True)
+    clean(project, build_system)
     expose_deps(project, build_system, dependency_type)
     pkg_version = get_package_version(project)
     os.environ[
@@ -143,9 +194,9 @@ def conda_build(project: str, dependency_type: str) -> None:
     set_up(project, build_system=B_CONDA, dependency_type=dependency_type)
 
     def _mk_conda_channel_arg(_project):
-        return f"""-c "{pathlib.Path(make_build_path(_project)).as_uri()}" """
+        return f"""-c "{pathlib.Path(make_build_path(_project,B_CONDA)).as_uri()}" """
 
-    build_path = make_build_path(project)
+    build_path = make_build_path(project, B_CONDA)
     os.environ[CONDA_BUILD_PATH_ENV] = build_path
 
     recipe_path = os.path.abspath(
@@ -155,8 +206,10 @@ def _mk_conda_channel_arg(_project):
     local_channels = [
         _mk_conda_channel_arg(p)
         for p in KNOWN_PROJECTS
-        if os.path.exists(make_build_path(p))
-        and os.path.exists(os.path.join(make_build_path(p), "noarch/repodata.json"))
+        if os.path.exists(make_build_path(p, B_CONDA))
+        and os.path.exists(
+            os.path.join(make_build_path(p, B_CONDA), "noarch/repodata.json")
+        )
     ]
 
     print(f"Building: {project}. Build path: {build_path}")
@@ -179,6 +232,43 @@ def tox_build(project: str, dependency_type: str) -> None:
     build_cmd = f"tox -e {tox_env} -v"
     print(f"Build Command: {build_cmd}")
     subprocess.run(args=build_cmd, shell=True, check=True)
+    print("Tox build completed – creating local PyPi index")
+    create_local_pypi_index(project)
+
+
+def create_local_pypi_index(project: str) -> None:
+    """
+    Creates/updates a local PyPI PEP 503 (the simple repository API) compliant
+    folder structure, so that it can be used with PIP's --extra-index-url
+    setting.
+    """
+    main_tox_build_path = make_build_path(project, B_TOX)
+    pypi_index_path = make_local_pypi_index_path(project)
+    project_dist_name = get_package_dist_name(project)
+    project_repo_path = os.path.join(pypi_index_path, project_dist_name)
+    project_index_html_path = os.path.join(project_repo_path, "index.html")
+    os.makedirs(project_repo_path, exist_ok=True)
+
+    package_glob = f"{project_dist_name}-*.tar.gz"
+
+    # copy all relevant packages into the index subfolder
+    for package in glob(os.path.join(main_tox_build_path, package_glob)):
+        shutil.copy(package, project_repo_path)
+
+    # remove index.html, if exists already
+    if os.path.exists(project_index_html_path):
+        os.remove(project_index_html_path)
+
+    # create an index.html with entries for all existing packages
+    package_file_links = [
+        f"<a href='{os.path.basename(package)}'>{os.path.basename(package)}</a><br />"
+        for package in glob(os.path.join(project_repo_path, package_glob))
+    ]
+    # store index.html
+    with open(project_index_html_path, "wt") as f:
+        f.writelines(package_file_links)
+
+    print(f"Local PyPi Index created at: {pypi_index_path}")
 
 
 def print_usage() -> None:

tox.ini

@@ -7,6 +7,7 @@ envlist = py3,
 skip_missing_interpreters = true
 isolated_build = true
 minversion = 3.7
+distshare= {toxinidir}/dist/tox
 
 [testenv]
 changedir = test