CHIA-597: Use poetry for package management

.coveragerc

@@ -13,7 +13,7 @@ parallel=True
 [paths]
 source =
   chia/
-  venv/**/site-packages/chia/
+  .venv/**/site-packages/chia/
 
 [report]
 precision = 1

.github/workflows/check_wheel_availability.yaml

@@ -67,5 +67,15 @@ jobs:
           python-version: ${{ matrix.python-version }}
           force-pyenv: ${{ matrix.os.matrix == 'macos' && matrix.arch.matrix == 'intel' }}
 
+      - name: Setup Poetry
+        if: matrix.os.matrix != 'windows'
+        run: |
+          sh setup-poetry.sh ${{ matrix.python-version }}
+
+      - name: Setup Poetry
+        if: matrix.os.matrix == 'windows'
+        run: |
+          ./Setup-poetry.ps1 ${{ matrix.python-version }}
+
       - name: Check Wheel Availability
         run: python build_scripts/check_dependency_artifacts.py

.github/workflows/dependency-review.yml

@@ -21,4 +21,5 @@ jobs:
       - name: "Dependency Review"
         uses: actions/dependency-review-action@v4
         with:
+          allow-dependencies-licenses: pkg:pypi/pylint
           deny-licenses: AGPL-1.0-only, AGPL-1.0-or-later, AGPL-1.0-or-later, AGPL-3.0-or-later, GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later

.github/workflows/reflow-version.yml

@@ -40,9 +40,11 @@ jobs:
 
       - name: Create installer version number
         id: version-number
+        # TODO figure out better way to handle versioning
         run: |
           python3 -m venv ../venv
           . ../venv/bin/activate
-          pip install setuptools_scm
-          echo "chia-installer-version=$(python3 ./build_scripts/installer-version.py)" >> "$GITHUB_OUTPUT"
+          python -m pip install --upgrade pip
+          pip install poetry "poetry-dynamic-versioning[plugin]"
+          echo "chia-installer-version=$(poetry version -s)" >> "$GITHUB_OUTPUT"
           deactivate

.github/workflows/test-single.yml

@@ -228,7 +228,7 @@ jobs:
           echo '==== collecting installed tests ===='
           if pytest --collect-only -qq --import-mode=append --pyargs chia._tests > installed_tests_raw
           then
-            cat installed_tests_raw | sed -n 's;^venv/.*/\(chia/_tests/.*\);\1;p' | sort > installed_tests
+            cat installed_tests_raw | sed -n 's;^\.venv/.*/\(chia/_tests/.*\);\1;p' | sort > installed_tests
           else
             echo '    ==== installed test collection failure'
             cat installed_tests_raw

.github/workflows/upload-pypi-source.yml

@@ -165,11 +165,16 @@ jobs:
 
   upload_source_dist:
     name: Lint and Upload source distribution
-    runs-on: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     needs:
       - mypy
       - check
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+
     steps:
       - name: Add safe git directory
         uses: chia-network/actions/git-mark-workspace-safe@main
@@ -194,6 +199,7 @@ jobs:
 
       - name: Build source distribution
         run: |
+          # TODO: let poetry do this?
           python -m build --sdist --outdir dist .
 
       - name: Upload artifacts

.gitignore

@@ -41,7 +41,10 @@ build_scripts/build
 
 # pyenv
 .python-version
+.penv*/
+.venv*/
 venv*/
+venv*
 activate
 
 # Editors

Install.ps1

@@ -10,17 +10,12 @@ param(
 $ErrorActionPreference = "Stop"
 
 $extras = @()
+$extras += "upnp"
 if ($d)
 {
     $extras += "dev"
 }
 
-$pip_parameters = @()
-if (-not $i)
-{
-    $pip_parameters += "--editable"
-}
-
 if ([Environment]::Is64BitOperatingSystem -eq $false)
 {
     Write-Output "Chia requires a 64-bit Windows installation"
@@ -100,31 +95,36 @@ if ($openSSLVersion -lt 269488367)
     Write-Output "Anything before 1.1.1n is vulnerable to CVE-2022-0778."
 }
 
-if ($extras.length -gt 0)
+$extras_cli = @()
+foreach ($extra in $extras)
 {
-    $extras_cli = $extras -join ","
-    $pip_parameters += ".[$extras_cli]"
-}
-else
-{
-    $pip_parameters += "."
+    $extras_cli += "--extras"
+    $extras_cli += $extra
 }
 
-py -$pythonVersion -m venv venv
+./Setup-poetry.ps1 -pythonVersion "$pythonVersion"
+.penv/Scripts/poetry env use $(py -"$pythonVersion" -c 'import sys; print(sys.executable)')
+# TODO: Decide if this is needed or should be handled automatically in some way
+.penv/Scripts/pip install "poetry-dynamic-versioning[plugin]"
+.penv/Scripts/poetry install @extras_cli
 
-venv\scripts\python -m pip install --upgrade pip setuptools wheel
-venv\scripts\pip install --extra-index-url https://pypi.chia.net/simple/ miniupnpc==2.2.2
-& venv\scripts\pip install @pip_parameters --extra-index-url https://pypi.chia.net/simple/
+if ($i)
+{
+    Write-Output "Running 'pip install --no-deps .' for non-editable"
+    .venv/Scripts/python -m pip install --no-deps .
+}
 
 if ($p)
 {
     $PREV_VIRTUAL_ENV = "$env:VIRTUAL_ENV"
-    $env:VIRTUAL_ENV = "venv"
+    $env:VIRTUAL_ENV = ".venv"
     .\Install-plotter.ps1 bladebit
     .\Install-plotter.ps1 madmax
     $env:VIRTUAL_ENV = "$PREV_VIRTUAL_ENV"
 }
 
+cmd /c mklink /j venv .venv
+
 Write-Output ""
 Write-Output "Chia blockchain .\Install.ps1 complete."
 Write-Output "For assistance join us on Discord in the #support chat channel:"
@@ -133,6 +133,6 @@ Write-Output ""
 Write-Output "Try the Quick Start Guide to running chia-blockchain:"
 Write-Output "https://github.com/Chia-Network/chia-blockchain/wiki/Quick-Start-Guide"
 Write-Output ""
-Write-Output "To install the GUI run '.\venv\scripts\Activate.ps1' then '.\Install-gui.ps1'."
+Write-Output "To install the GUI run '.\.venv\scripts\Activate.ps1' then '.\Install-gui.ps1'."
 Write-Output ""
-Write-Output "Type '.\venv\Scripts\Activate.ps1' and then 'chia init' to begin."
+Write-Output "Type '.\.venv\Scripts\Activate.ps1' and then 'chia init' to begin."

Setup-poetry.ps1

@@ -0,0 +1,12 @@
+param(
+    [Parameter(Mandatory, HelpMessage="Python version")]
+    [string]
+    $pythonVersion
+)
+
+$ErrorActionPreference = "Stop"
+
+py -$pythonVersion -m venv .penv
+.penv/Scripts/python -m pip install --upgrade pip setuptools wheel
+# TODO: maybe make our own zipapp/shiv/pex of poetry and download that?
+.penv/Scripts/python -m pip install poetry

activated.ps1

@@ -6,7 +6,7 @@ $command = $args[0]
 $parameters = [System.Collections.ArrayList]$args
 $parameters.RemoveAt(0)
 
-& $script_directory/venv/Scripts/Activate.ps1
+& $script_directory/.venv/Scripts/Activate.ps1
 & $command @parameters
 
 exit $LASTEXITCODE

activated.sh

@@ -7,6 +7,6 @@ SCRIPT_DIRECTORY=$(
   pwd
 )
 # shellcheck disable=SC1091
-. "${SCRIPT_DIRECTORY}/venv/bin/activate"
+. "${SCRIPT_DIRECTORY}/.venv/bin/activate"
 
 "$@"

build_scripts/check_dependency_artifacts.py

@@ -16,6 +16,10 @@
 }
 
 
+here = pathlib.Path(__file__).parent
+project_root = here.parent
+
+
 def excepted(path: pathlib.Path) -> bool:
     # TODO: This should be implemented with a real file name parser though i'm
     #       uncertain at the moment what package that would be.
@@ -29,14 +33,10 @@ def main() -> int:
         print(f"Working in: {directory_string}")
         print()
         directory_path = pathlib.Path(directory_string)
+        artifact_directory_path = directory_path.joinpath("artifacts")
+        artifact_directory_path.mkdir()
 
         extras = ["upnp"]
-        package_path_string = os.fspath(pathlib.Path(__file__).parent.parent)
-
-        if len(extras) > 0:
-            package_and_extras = f"{package_path_string}[{','.join(extras)}]"
-        else:
-            package_and_extras = package_path_string
 
         print("Downloading packages for Python version:")
         lines = [
@@ -49,24 +49,53 @@ def main() -> int:
             print(f"    {line}")
         print(flush=True)
 
+        requirements_path = directory_path.joinpath("exported_requirements.txt")
+
+        if sys.platform == "win32":
+            poetry_path = pathlib.Path(".penv/Scripts/poetry")
+        else:
+            poetry_path = pathlib.Path(".penv/bin/poetry")
+
+        poetry_path = project_root.joinpath(poetry_path)
+
+        subprocess.run(
+            [
+                os.fspath(poetry_path),
+                "export",
+                "--format",
+                "requirements.txt",
+                "--output",
+                os.fspath(requirements_path),
+                "--without-hashes",
+                "--no-ansi",
+                "--no-interaction",
+                *(f"--extras={extra}" for extra in extras),
+            ],
+            check=True,
+        )
+
+        env = {key: value for key, value in os.environ.items() if key != "PIP_REQUIRE_VIRTUALENV"}
+
         subprocess.run(
             [
                 sys.executable,
                 "-m",
                 "pip",
                 "download",
                 "--dest",
-                os.fspath(directory_path),
+                os.fspath(artifact_directory_path),
                 "--extra-index",
                 "https://pypi.chia.net/simple/",
-                package_and_extras,
+                "--requirement",
+                os.fspath(requirements_path),
             ],
+            env=env,
             check=True,
         )
 
         failed_artifacts = []
 
-        for artifact in directory_path.iterdir():
+        for artifact in artifact_directory_path.iterdir():
             if artifact.suffix == ".whl":
                 # everything being a wheel is the target
                 continue

build_scripts/pyinstaller.spec

@@ -10,7 +10,7 @@ from PyInstaller.utils.hooks import collect_submodules, copy_metadata
 THIS_IS_WINDOWS = platform.system().lower().startswith("win")
 THIS_IS_MAC = platform.system().lower().startswith("darwin")
 
-ROOT = pathlib.Path(importlib.import_module("chia").__file__).absolute().parent.parent
+ROOT = pathlib.Path(SPECPATH).absolute().parent
 
 
 keyring_imports = collect_submodules("keyring.backends")

chia/_tests/core/data_layer/util.py

@@ -4,6 +4,7 @@
 import functools
 import os
 import pathlib
+import shutil
 import subprocess
 from dataclasses import dataclass
 from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Union, overload
@@ -137,8 +138,11 @@ def run(
         if "SYSTEMROOT" in os.environ:
             kwargs["env"]["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
 
+        chia_executable = shutil.which("chia")
+        if chia_executable is None:
+            chia_executable = "chia"
         modified_args: List[Union[str, os_PathLike_str]] = [
-            self.scripts_path.joinpath("chia"),
+            self.scripts_path.joinpath(chia_executable),
             "--root-path",
             self.path,
             *args,
@@ -151,7 +155,10 @@ def run(
         kwargs["stdout"] = stdout
         kwargs["stderr"] = stderr
 
-        return subprocess.run(*final_args, **kwargs)
+        try:
+            return subprocess.run(*final_args, **kwargs)
+        except OSError as e:
+            raise Exception(f"failed to run:\n    {final_args}\n    {kwargs}") from e
 
     def read_log(self) -> str:
         return self.path.joinpath("log", "debug.log").read_text(encoding="utf-8")

chia/_tests/util/test_installed.py

@@ -17,4 +17,4 @@ def test_chia_installed() -> None:
     and testing the installed code checks for that.  A next step would be to install
     using the actual wheel file we are going to publish.
     """
-    assert "venv" in pathlib.Path(chia.__file__).parts
+    assert ".venv" in pathlib.Path(chia.__file__).parts