From 9fb971d72928d8258db7882b202f8db589f7c15b Mon Sep 17 00:00:00 2001 From: juftin Date: Thu, 9 Nov 2023 23:08:59 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20use=20pip-tools=20cli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/publish.yaml | 34 +++++ README.md | 114 ++++++++++++++++ docs/index.md | 118 ++++++++++++++++ hatch_pip_compile/plugin.py | 238 ++++++++++++++++----------------- pyproject.toml | 1 + 5 files changed, 382 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..a344114 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,34 @@ +name: Publishing + +on: + release: + types: + - published + +jobs: + pypi-publish: + name: PyPI + if: github.repository_owner == 'juftin' + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install Hatch + run: | + python -m pip install --upgrade pip + python -m pip install -q hatch pre-commit + hatch --version + - name: Build package + run: | + hatch build + - name: Publish package on PyPI + uses: pypa/gh-action-pypi-publish@v1.8.10 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} diff --git a/README.md b/README.md index c77301b..6e1e111 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,125 @@ hatch plugin to use pip-compile to manage project dependencies +[![PyPI](https://img.shields.io/pypi/v/hatch-pip-compile?color=blue&label=🔨%20hatch-pip-compile)](https://github.com/juftin/hatch-pip-compile) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hatch-pip-compile)](https://pypi.python.org/pypi/hatch-pip-compile/) +[![GitHub License](https://img.shields.io/github/license/juftin/hatch-pip-compile?color=blue&label=License)](https://github.com/juftin/hatch-pip-compile/blob/main/LICENSE) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-lightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) [![Gitmoji](https://img.shields.io/badge/gitmoji-%20😜%20😍-FFDD67.svg)](https://gitmoji.dev) +## Installation + +```shell +pip install hatch-pip-compile +``` + +### pipx + +Personally, I use [pipx](https://github.com/pypa/pipx) to install and use hatch. If you do too, +you will need to inject the `hatch-pip-compile` plugin into the hatch environment. + +```shell +pipx install hatch +pipx inject hatch hatch-pip-compile +``` + +## Usage + +The `hatch-pip-compile` plugin will automatically run `pip-compile` whenever your +environment needs to be updated. Behind the scenes, this plugin creates a lockfile +at `.hatch/.lock`. Alongside `pip-compile`, this plugin also uses +`pip-sync` to install the dependencies from the lockfile into your environment. + +## Configuration + +The [environment plugin](https://hatch.pypa.io/latest/plugins/environment/) name is `pip-compile`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + ``` + +### lock-directory + +The directory where the lockfiles will be stored. Defaults to `.hatch`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + lock-directory = "requirements" + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + lock-directory = "requirements" + ``` + +### pip-compile-hashes + +Whether or not to use hashes in the lockfile. Defaults to `true`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + pip-compile-hashes = true + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + pip-compile-hashes = true + ``` + +### pip-compile-args + +Extra arguments to pass to `pip-compile`. Defaults to None. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + pip-compile-args = [ + "--index-url", + "https://pypi.org/simple", + ] + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + pip-compile-args = [ + "--index-url", + "https://pypi.org/simple", + ] + ``` + +--- + +--- + #### Check Out the [Docs](https://juftin.github.io/hatch-pip-compile/) #### Looking to contribute? See the [Contributing Guide](https://juftin.github.io/hatch-pip-compile/contributing) diff --git a/docs/index.md b/docs/index.md index 61c2c08..a5314a2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,125 @@ hatch plugin to use pip-compile to manage project dependencies +[![PyPI](https://img.shields.io/pypi/v/hatch-pip-compile?color=blue&label=🔨%20hatch-pip-compile)](https://github.com/juftin/hatch-pip-compile) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hatch-pip-compile)](https://pypi.python.org/pypi/hatch-pip-compile/) +[![GitHub License](https://img.shields.io/github/license/juftin/hatch-pip-compile?color=blue&label=License)](https://github.com/juftin/hatch-pip-compile/blob/main/LICENSE) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-lightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) [![Gitmoji](https://img.shields.io/badge/gitmoji-%20😜%20😍-FFDD67.svg)](https://gitmoji.dev) + +## Installation + +```shell +pip install hatch-pip-compile +``` + +### pipx + +Personally, I use [pipx](https://github.com/pypa/pipx) to install and use hatch. If you do too, +you will need to inject the `hatch-pip-compile` plugin into the hatch environment. + +```shell +pipx install hatch +pipx inject hatch hatch-pip-compile +``` + +## Usage + +The `hatch-pip-compile` plugin will automatically run `pip-compile` whenever your +environment needs to be updated. Behind the scenes, this plugin creates a lockfile +at `.hatch/.lock`. Alongside `pip-compile`, this plugin also uses +`pip-sync` to install the dependencies from the lockfile into your environment. + +## Configuration + +The [environment plugin](https://hatch.pypa.io/latest/plugins/environment/) name is `pip-compile`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + ``` + +### lock-directory + +The directory where the lockfiles will be stored. Defaults to `.hatch`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + lock-directory = "requirements" + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + lock-directory = "requirements" + ``` + +### pip-compile-hashes + +Whether or not to use hashes in the lockfile. Defaults to `true`. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + pip-compile-hashes = true + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + pip-compile-hashes = true + ``` + +### pip-compile-args + +Extra arguments to pass to `pip-compile`. Defaults to None. + +- **_pyproject.toml_** + + ```toml + [tool.hatch.envs.] + type = "pip-compile" + pip-compile-args = [ + "--index-url", + "https://pypi.org/simple", + ] + ``` + +- **_hatch.toml_** + + ```toml + [envs.] + type = "pip-compile" + pip-compile-args = [ + "--index-url", + "https://pypi.org/simple", + ] + ``` + +--- + +--- + +
+ +

logo

diff --git a/hatch_pip_compile/plugin.py b/hatch_pip_compile/plugin.py index 488c462..40ec0e0 100644 --- a/hatch_pip_compile/plugin.py +++ b/hatch_pip_compile/plugin.py @@ -5,25 +5,11 @@ from __future__ import annotations import pathlib +import re import tempfile -from typing import Any, ClassVar +from textwrap import dedent from hatch.env.virtual import VirtualEnvironment -from pip._internal.req import InstallRequirement -from piptools._compat import parse_requirements -from piptools.cache import DependencyCache -from piptools.locations import CACHE_DIR -from piptools.repositories import PyPIRepository -from piptools.resolver import BacktrackingResolver -from piptools.writer import OutputWriter - - -class DummyContext: - """ - Dummy Context for Click - """ - - params: ClassVar[dict[str, Any]] = {} class PipCompileEnvironment(VirtualEnvironment): @@ -33,125 +19,87 @@ class PipCompileEnvironment(VirtualEnvironment): PLUGIN_NAME = "pip-compile" - def _get_piptools_repo(self) -> PyPIRepository: + def __init__(self, *args, **kwargs): """ - Get a pip-tools PyPIRepository instance + Initialize PipCompileEnvironment with extra attributes """ - return PyPIRepository(pip_args=[], cache_dir=CACHE_DIR) + super().__init__(*args, **kwargs) + self._piptools_lock_file = self._config_lock_directory / f"{self.name}.lock" - def _get_piptools_requirements(self, repository: PyPIRepository) -> list[InstallRequirement]: + @staticmethod + def get_option_types(): """ - Generate Requirements + Get option types """ - with tempfile.TemporaryDirectory() as temp_dir: - requirements_file = pathlib.Path(temp_dir) / "requirements.in" - requirements_file.write_text("\n".join(self.dependencies)) - requirements = list( - parse_requirements( - filename=str(requirements_file), - session=repository.session, - finder=repository.finder, - options=repository.options, - ) - ) - return requirements + return {"lock-directory": str, "pip-compile-args": list[str], "pip-compile-hashes": bool} - def _get_piptools_resolver( - self, repository: PyPIRepository, requirements: list[InstallRequirement] - ) -> BacktrackingResolver: + @property + def _config_lock_directory(self) -> pathlib.Path: """ - Get a pip-tools BacktrackingResolver instance + Get the lock directory from the config """ - return BacktrackingResolver( - constraints=requirements, - existing_constraints={}, - repository=repository, - prereleases=False, - cache=DependencyCache(CACHE_DIR), - clear_caches=False, - allow_unsafe=False, - unsafe_packages=None, - ) + default_lock_dir = self.root / ".hatch" + lock_dir = self.config.get("lock-directory", default_lock_dir) + return pathlib.Path(lock_dir) - def _get_piptools_results( - self, resolver: BacktrackingResolver - ) -> tuple[set[InstallRequirement], dict[InstallRequirement, set[str]]]: - """ - Fetch pip-tools results - """ - results = resolver.resolve(max_rounds=10) - hashes = resolver.resolve_hashes(results) - return results, hashes - - def _write_piptools_results( - self, - repository: PyPIRepository, - results: set[InstallRequirement], - hashes: dict[InstallRequirement, set[str]], - resolver: BacktrackingResolver, - ) -> None: - """ - Write pip-tools results to requirements.txt - """ - output_file = self.root / "requirements.txt" - writer = OutputWriter( - dst_file=output_file.open("wb"), - click_ctx=DummyContext(), # type: ignore - dry_run=False, - emit_header=True, - emit_index_url=False, - emit_trusted_host=False, - annotate=False, - annotation_style="split", - strip_extras=True, - generate_hashes=True, - default_index_url=repository.DEFAULT_INDEX_URL, - index_urls=repository.finder.index_urls, - trusted_hosts=repository.finder.trusted_hosts, - format_control=repository.finder.format_control, - linesep="\n", - allow_unsafe=False, - find_links=repository.finder.find_links, - emit_find_links=False, - emit_options=False, - ) - writer.write( - results=results, - unsafe_packages=resolver.unsafe_packages, - unsafe_requirements=resolver.unsafe_constraints, - markers={}, - hashes=hashes, - ) + def _pip_compile_command(self, output_file: pathlib.Path, input_file: pathlib.Path) -> None: + """ + Run pip-compile + """ + self.platform.check_command(self.construct_pip_install_command(["pip-tools"])) + cmd = [ + self.virtual_env.python_info.executable, + "-m", + "piptools", + "compile", + "--quiet", + "--strip-extras", + "--no-header", + "--output-file", + str(output_file), + "--resolver=backtracking", + ] + if self.config.get("pip-compile-hashes", True) is True: + cmd.append("--generate-hashes") + cmd.extend(self.config.get("pip-compile-args", [])) + cmd.append(str(input_file)) + self.platform.check_command(cmd) + self._post_process_lockfile() - def _pip_compile(self) -> None: + def _pip_compile_cli(self) -> None: """ - Compile requirements.txt + Run pip-compile """ - repository = self._get_piptools_repo() - requirements = self._get_piptools_requirements(repository=repository) - resolver = self._get_piptools_resolver(repository=repository, requirements=requirements) - results, hashes = self._get_piptools_results(resolver=resolver) - self._write_piptools_results( - repository=repository, results=results, hashes=hashes, resolver=resolver - ) + self._config_lock_directory.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = pathlib.Path(tmpdir) + input_file = tmp_path / f"{self.name}.in" + input_file.write_text("\n".join([*self.dependencies, ""])) + self._pip_compile_command(output_file=self._piptools_lock_file, input_file=input_file) - def install_project(self): - msg = "Project must be installed in dev mode in pip-compile environments." - raise NotImplementedError(msg) + def _root_requirements(self) -> None: + """ + Run pip-compile + """ + self._config_lock_directory.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = pathlib.Path(tmpdir) + input_file = tmp_path / f"{self.name}.in" + input_file.write_text("\n".join([*self.dependencies, ""])) + self._pip_compile_command(output_file=self._piptools_lock_file, input_file=input_file) - def _pip_install_piptools(self) -> None: + def _pip_sync_cli(self) -> None: """ - Install pip-tools + Run pip-sync """ - output_file = self.root / "requirements.txt" cmd = [ - "python", - "-u", + self.virtual_env.python_info.executable, "-m", - "pip", - "install", - "-r", - str(output_file), + "piptools", + "sync", + "--python-executable", + str(self.virtual_env.python_info.executable), + str(self._piptools_lock_file), ] self.platform.check_command(cmd) @@ -160,19 +108,63 @@ def install_project_dev_mode(self): Install the project the first time in dev mode """ with self.safe_activation(): - self._pip_compile() - self._pip_install_piptools() + self._pip_compile_cli() + self._pip_sync_cli() def dependencies_in_sync(self): """ - Always return False to force sync_dependencies + Handle whether dependencies should be synced """ - return False + if len(self.dependencies) > 0 and (self._piptools_lock_file.exists() is False): + return False + elif len(self.dependencies) > 0 and (self._piptools_lock_file.exists() is True): + expected_locks = self._lock_file_compare() + if expected_locks is False: + return False + return super().dependencies_in_sync() def sync_dependencies(self): """ Sync dependencies """ with self.safe_activation(): - self._pip_compile() - self._pip_install_piptools() + self._pip_compile_cli() + self._pip_sync_cli() + + def _post_process_lockfile(self) -> None: + """ + Post process lockfile + """ + joined_dependencies = "\n # - ".join(self.dependencies) + prefix = f""" + #################################################################################### + # 🔒 hatch-pip-compile 🔒 + # + # - {joined_dependencies} + # + #################################################################################### + """ + lockfile_text = self._piptools_lock_file.read_text() + new_data = re.sub( + rf"-r \S*/{self.name}\.in", + f"{self.metadata.name} (pyproject.toml)", + lockfile_text, + ) + new_text = dedent(prefix).strip() + "\n\n" + new_data + self._piptools_lock_file.write_text(new_text) + + def _lock_file_compare(self) -> bool: + """ + Compare lock file + """ + lock_file_text = self._piptools_lock_file.read_text() + parsed_requirements = [] + for line in lock_file_text.splitlines(): + if line.startswith("# - "): + rest_of_line = line[4:] + parsed_requirements.append(rest_of_line) + elif not line.startswith("#"): + break + expected_output = "\n".join([*parsed_requirements, ""]) + new_output = "\n".join([*self.dependencies, ""]) + return expected_output == new_output diff --git a/pyproject.toml b/pyproject.toml index 79ec504..308351d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ authors = [ ] classifiers = [ "Development Status :: 3 - Alpha", + "Framework :: Hatch", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3.8",