From 5d5f372ba608c972f9dcb82ce349e2f6c9a1408b Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Fri, 28 Jun 2024 14:30:25 -0500 Subject: [PATCH] feat(back): #1345 make python poetry environment - Add makePythonPoetryEnvironment builtin - Make makePythonEnvironment use such builtin - Add documentation - deprecate listOptional --- docs/src/api/extensions/python.md | 73 ++++++++++++++++- src/args/agnostic.nix | 3 +- src/args/make-python-environment/default.nix | 78 ++++--------------- .../default.nix | 64 +++++++++++++++ src/evaluator/modules/cache/default.nix | 4 +- 5 files changed, 152 insertions(+), 70 deletions(-) create mode 100644 src/args/make-python-poetry-environment/default.nix diff --git a/docs/src/api/extensions/python.md b/docs/src/api/extensions/python.md index b6e7bfd6..d203785c 100644 --- a/docs/src/api/extensions/python.md +++ b/docs/src/api/extensions/python.md @@ -1,7 +1,7 @@ ## makePythonEnvironment -Create a Python virtual environment using -[poetry2nix](https://github.com/nix-community/poetry2nix/tree/74921da7e0cc8918adc2e9989bd3e9c127b25ff6). +Create a source-able Python virtual environment using +[makePythonPoetryEnvironment](#makepythonpoetryenvironment). Pre-requisites: Having both `pyproject.toml` and `poetry.lock`. @@ -67,6 +67,75 @@ Example: Refer to [makePythonLock](/api/builtins/utilities/#makepythonlock) to learn how to generate a `poetry.lock`. +## makePythonPoetryEnvironment + +Create a Python virtual environment using +[poetry2nix](https://github.com/nix-community/poetry2nix/tree/74921da7e0cc8918adc2e9989bd3e9c127b25ff6). + +Pre-requisites: +Having both `pyproject.toml` and `poetry.lock`. + +Types: + +- makePythonPoetryEnvironment: (`function { ... } -> poetry2nixBundle`): + - pythonProjectDir (`path`): Required. + Python project where both + `pyproject.toml` and `poetry.lock` + are located. + - pythonVersion (`str`): Required. + Python version used to build the environment. + Supported versions are `3.9`, `3.10`, `3.11` and `3.12`. + - preferWheels (`bool`): Optional. + Use pre-compiled wheels from PyPI. + Defaults to `true`. + - overrides (`function {...} -> package`): Optional. + Override build attributes for libraries within the environment. + For more information see [here](https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md). + Defaults to `(self: super: {})`. + + ???+ note + + By default we override every python package deleting the + `homeless-shelter` directory and changing the `HOME` variable, + we make this to assure purity of builds without sandboxing. + +Example: + +=== "main.nix" + + ```nix + # /path/to/my/project/makes/example/main.nix + { + makePythonPoetryEnvironment, + projectPath, + ... + }: + makePythonPoetryEnvironment { + pythonProjectDir = projectPath "/makes/example"; + pythonVersion = "3.11"; + preferWheels = true; + # Consider pygments requiring setuptools to build properly + overrides = self: super: { + pygments = super.pygments.overridePythonAttrs ( + old: { + preUnpack = + '' + export HOME=$(mktemp -d) + rm -rf /homeless-shelter + '' + + (old.preUnpack or ""); + buildInputs = [super.setuptools]; + } + ); + }; + } + ``` + +???+ tip + + Refer to [makePythonLock](/api/builtins/utilities/#makepythonlock) + to learn how to generate a `poetry.lock`. + ## makePythonPyprojectPackage Create a python package bundle using nixpkgs build functions. diff --git a/src/args/agnostic.nix b/src/args/agnostic.nix index e6443a68..0d64af77 100644 --- a/src/args/agnostic.nix +++ b/src/args/agnostic.nix @@ -58,7 +58,6 @@ let inherit (__nixpkgs__.stdenv) isDarwin; inherit (__nixpkgs__.stdenv) isLinux; libGit = import ./lib-git/default.nix self; - listOptional = __nixpkgs__.lib.lists.optional; lintClojure = import ./lint-clojure/default.nix self; lintGitCommitMsg = import ./lint-git-commit-msg/default.nix self; lintGitMailMap = import ./lint-git-mailmap/default.nix self; @@ -83,6 +82,8 @@ let makeNodeJsModules = import ./make-node-js-modules/default.nix self; makeNomadEnvironment = import ./make-nomad-environment/default.nix self; makePythonEnvironment = import ./make-python-environment/default.nix self; + makePythonPoetryEnvironment = + import ./make-python-poetry-environment/default.nix self; makePythonPyprojectPackage = import ./make-python-pyproject-package/default.nix; makePythonVscodeSettings = diff --git a/src/args/make-python-environment/default.nix b/src/args/make-python-environment/default.nix index 25f5a7c4..9c603c16 100644 --- a/src/args/make-python-environment/default.nix +++ b/src/args/make-python-environment/default.nix @@ -1,73 +1,21 @@ -{ __nixpkgs__, listOptional, makeSearchPaths, ... }: +{ __nixpkgs__, makePythonPoetryEnvironment, makeSearchPaths, ... }: { pythonProjectDir, pythonVersion, preferWheels ? true , overrides ? (self: super: { }), }: let - # Import poetry2nix - poetry2nix = let - commit = "528d500ea826383cc126a9be1e633fc92b19ce5d"; - sha256 = "sha256:1q245v4q0bb30ncfj66gl6dl1k46am28x7kjj6d3y7r6l4fzppq8"; - src = builtins.fetchTarball { - inherit sha256; - url = - "https://api.github.com/repos/nix-community/poetry2nix/tarball/${commit}"; - }; - in import src { pkgs = __nixpkgs__; }; - - # Decide Python version - is39 = pythonVersion == "3.9"; - is310 = pythonVersion == "3.10"; - is311 = pythonVersion == "3.11"; - is312 = pythonVersion == "3.12"; - python = { - "3.9" = __nixpkgs__.python39; - "3.10" = __nixpkgs__.python310; - "3.11" = __nixpkgs__.python311; - "3.12" = __nixpkgs__.python312; - }.${pythonVersion}; - - # Override HOME directory for each package/derivation - # and add any custom override on top of it - overridenPackages = self: super: - let - overridePackageHome = pkg: super: - super.${pkg}.overridePythonAttrs (old: { - preUnpack = '' - export HOME=$(mktemp -d) - rm -rf /homeless-shelter - '' + (old.preUnpack or ""); - }); - packages = let - lock = "${pythonProjectDir}/poetry.lock"; - data = builtins.fromTOML (builtins.readFile lock); - main = builtins.map (pkg: pkg.name) data.package; - extras = let - list = builtins.concatLists (builtins.map (pkg: - if builtins.hasAttr "extras" pkg then - builtins.concatLists (builtins.attrValues pkg.extras) - else - [ ]) data.package); - names = builtins.map (extra: - builtins.toString (builtins.head - (builtins.split " " (__nixpkgs__.lib.toLower extra)))) list; - in names; - in main ++ extras; - overridenHomes = builtins.listToAttrs (builtins.map (pkg: { - name = pkg; - value = overridePackageHome pkg super; - }) packages); - in overridenHomes // (overrides self super); - - # Final python environment - env = poetry2nix.mkPoetryEnv { - overrides = poetry2nix.defaultPoetryOverrides.extend overridenPackages; + env = makePythonPoetryEnvironment { + inherit pythonProjectDir; + inherit pythonVersion; inherit preferWheels; - projectDir = pythonProjectDir; - inherit python; + inherit overrides; }; in makeSearchPaths { bin = [ env ]; - pythonPackage39 = listOptional is39 env; - pythonPackage310 = listOptional is310 env; - pythonPackage311 = listOptional is311 env; - pythonPackage312 = listOptional is312 env; + pythonPackage39 = + __nixpkgs__.lib.lists.optional (env.pythonVersion == "3.9") env; + pythonPackage310 = + __nixpkgs__.lib.lists.optional (env.pythonVersion == "3.10") env; + pythonPackage311 = + __nixpkgs__.lib.lists.optional (env.pythonVersion == "3.11") env; + pythonPackage312 = + __nixpkgs__.lib.lists.optional (env.pythonVersion == "3.12") env; } diff --git a/src/args/make-python-poetry-environment/default.nix b/src/args/make-python-poetry-environment/default.nix new file mode 100644 index 00000000..ba51b272 --- /dev/null +++ b/src/args/make-python-poetry-environment/default.nix @@ -0,0 +1,64 @@ +{ __nixpkgs__, makeSearchPaths, ... }: +{ pythonProjectDir, pythonVersion, preferWheels ? true +, overrides ? (self: super: { }), }: +let + # Import poetry2nix + poetry2nix = let + commit = "528d500ea826383cc126a9be1e633fc92b19ce5d"; + sha256 = "sha256:1q245v4q0bb30ncfj66gl6dl1k46am28x7kjj6d3y7r6l4fzppq8"; + src = builtins.fetchTarball { + inherit sha256; + url = + "https://api.github.com/repos/nix-community/poetry2nix/tarball/${commit}"; + }; + in import src { pkgs = __nixpkgs__; }; + + # Decide Python version + is39 = pythonVersion == "3.9"; + is310 = pythonVersion == "3.10"; + is311 = pythonVersion == "3.11"; + is312 = pythonVersion == "3.12"; + python = { + "3.9" = __nixpkgs__.python39; + "3.10" = __nixpkgs__.python310; + "3.11" = __nixpkgs__.python311; + "3.12" = __nixpkgs__.python312; + }.${pythonVersion}; + + # Override HOME directory for each package/derivation + # and add any custom override on top of it + overridenPackages = self: super: + let + overridePackageHome = pkg: super: + super.${pkg}.overridePythonAttrs (old: { + preUnpack = '' + export HOME=$(mktemp -d) + rm -rf /homeless-shelter + '' + (old.preUnpack or ""); + }); + packages = let + lock = "${pythonProjectDir}/poetry.lock"; + data = builtins.fromTOML (builtins.readFile lock); + main = builtins.map (pkg: pkg.name) data.package; + extras = let + list = builtins.concatLists (builtins.map (pkg: + if builtins.hasAttr "extras" pkg then + builtins.concatLists (builtins.attrValues pkg.extras) + else + [ ]) data.package); + names = builtins.map (extra: + builtins.toString (builtins.head + (builtins.split " " (__nixpkgs__.lib.toLower extra)))) list; + in names; + in main ++ extras; + overridenHomes = builtins.listToAttrs (builtins.map (pkg: { + name = pkg; + value = overridePackageHome pkg super; + }) packages); + in overridenHomes // (overrides self super); +in poetry2nix.mkPoetryEnv { + overrides = poetry2nix.defaultPoetryOverrides.extend overridenPackages; + inherit preferWheels; + projectDir = pythonProjectDir; + inherit python; +} diff --git a/src/evaluator/modules/cache/default.nix b/src/evaluator/modules/cache/default.nix index dd8d67a5..93d03b83 100644 --- a/src/evaluator/modules/cache/default.nix +++ b/src/evaluator/modules/cache/default.nix @@ -1,4 +1,4 @@ -{ listOptional, ... }: +{ __nixpkgs__, ... }: { config, lib, ... }: { options = { cache = { @@ -36,7 +36,7 @@ config = { config = { cache = builtins.concatLists [ - (listOptional config.cache.readNixos { + (__nixpkgs__.lib.lists.optional config.cache.readNixos { url = "https://cache.nixos.org"; pubKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";