Skip to content

Commit

Permalink
feat(back): fluidattacks#1146 make python pyproject
Browse files Browse the repository at this point in the history
- add make-python-pyproject buiding from nixpkgs functions
- add python-override-utils
- add docs and example

Signed-off-by: Daniel F. Murcia Rivera <danmur97@outlook.com>
  • Loading branch information
danmur97 committed Sep 6, 2023
1 parent 45f7141 commit acf54c6
Show file tree
Hide file tree
Showing 11 changed files with 279 additions and 21 deletions.
111 changes: 111 additions & 0 deletions docs/src/api/extensions/python.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,114 @@ Example:

Refer to [makePythonLock](/api/builtins/utilities/#makepythonlock)
to learn how to generate a `sourcesYaml`.

## makePythonPyprojectPackage

Create a python package bundle using nixpkgs build functions.
This bundle includes the package itself, some modifications
over the tests and its python environments.

Types:

- makePythonPypiEnvironment: (`function Input -> Bundle`):
- Input: `AttrsOf`
- buildEnv: `function {...} -> Derivation`
The nixpkgs buildEnv.override function.
Commonly found at `nixpkgs."${python_version}".buildEnv.override`
- buildPythonPackage: `function {...} -> Derivation`
The nixpkgs buildPythonPackage function.
Commonly found at `nixpkgs."${python_version}".pkgs.buildPythonPackage`
- pkgDeps: `AttrsOf`
The package dependencies.
Usually other python packages build with nix,
but can be also a nix derivation of a binary.
- runtime_deps: `List[Derivation]`
- build_deps: `List[Derivation]`
- test_deps: `List[Derivation]`
- src: `NixPath`
The nix path to the source code of the python package.
i.e. not only be the package itself, it should also contain
a tests folder/module, the pyproject conf and any other meta-package
data that the build or tests requires (e.g. custom mypy conf).
- Bundle: `AttrsOf`
- check: `AttrsOf`
Builds of the package only including one test.
- tests:`Derivation`
- types: `Derivation`
- env: `AttrsOf`
- dev: `Derivation`
The python environment containing only
runtime_deps and test_deps
- runtime: `Derivation`
The python environment containing only
the package itself and its runtime_deps.
- pkg: `Derivation`
The output of the nixpkgs buildPythonPackage function
i.e. the python package

???+ tip

The default implemented tests require `mypy` and `pytest` as `test_deps`.
If you do not want the default, you can override the checkPhase
of the package i.e. using `pythonOverrideUtils` or using the
`overridePythonAttrs` function included on the derivation of
nix built python packages.

Example:

=== "main.nix"

```nix
# /path/to/my/project/makes/example/main.nix
{
inputs,
makeScript,
makePythonPyprojectPackage,
...
}: let
nixpkgs = inputs.nixpkgs;
python_version = "python311";
python_pkgs = nixpkgs."${python_version}Packages";
bundle = makePythonPyprojectPackage {
src = ./.;
buildEnv = nixpkgs."${python_version}".buildEnv.override;
buildPythonPackage = nixpkgs."${python_version}".pkgs.buildPythonPackage;
pkgDeps = {
runtime_deps = with python_pkgs; [click];
build_deps = with python_pkgs; [flit-core];
test_deps = with python_pkgs; [
mypy
pytest
];
};
};
env = bundle.env.runtime;
in
makeScript {
name = "my-cli";
searchPaths = {
bin = [
env
];
};
entrypoint = "my-cli \"\${@}\"";
# Assuming that the pyproject conf has
# a definition of `my-cli` as a cli entrypoint
}
```

???+ tip

Because env.runtime include the package,
all tests are triggered when building the environment.
If is desirable only to trigger an specific check phase,
then use the check derivations that override this phase.

???+ tip

To avoid performance issues use a shared cache
system (e.g. cachix) or an override over the package
to skip tests (unsafe way) to ensure that tests are
executed only once (or never).
This can also help on performance over heavy
compilation/build processes.
44 changes: 23 additions & 21 deletions src/args/agnostic.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
system ? builtins.currentSystem,
}: let
fix' = __unfix__: let x = __unfix__ x // {inherit __unfix__;}; in x;

args = fix' (self: {
sources = import ../nix/sources.nix;
args = fix' (self: let
__nixpkgs__ = import sources.nixpkgs {inherit system;};
in {
inherit __nixpkgs__;
__nixpkgsSrc__ = sources.nixpkgs;
__shellCommands__ = ./shell-commands/template.sh;
__shellOptions__ = ./shell-options/template.sh;
Expand All @@ -19,27 +21,27 @@
__toModuleOutputs__ = import ./to-module-outputs/default.nix self;
asContent = import ./as-content/default.nix;
attrsGet = import ./attrs-get/default.nix;
attrsMapToList = self.__nixpkgs__.lib.mapAttrsToList;
attrsMerge = builtins.foldl' self.__nixpkgs__.lib.recursiveUpdate {};
attrsOptional = self.__nixpkgs__.lib.optionalAttrs;
attrsMapToList = __nixpkgs__.lib.mapAttrsToList;
attrsMerge = builtins.foldl' __nixpkgs__.lib.recursiveUpdate {};
attrsOptional = __nixpkgs__.lib.optionalAttrs;
calculateCvss3 = import ./calculate-cvss-3/default.nix self;
calculateScorecard = import ./calculate-scorecard/default.nix self;
chunks = import ./chunks/default.nix self;
computeOnAwsBatch = import ./compute-on-aws-batch/default.nix self;
deployContainerImage = import ./deploy-container-image/default.nix self;
deployNomad = import ./deploy-nomad/default.nix self;
deployTerraform = import ./deploy-terraform/default.nix self;
inherit (self.__nixpkgs__.lib.strings) escapeShellArg;
inherit (self.__nixpkgs__.lib.strings) escapeShellArgs;
inherit (self.__nixpkgs__.lib) fakeSha256;
inherit (__nixpkgs__.lib.strings) escapeShellArg;
inherit (__nixpkgs__.lib.strings) escapeShellArgs;
inherit (__nixpkgs__.lib) fakeSha256;
fetchArchive = import ./fetch-archive/default.nix self;
fetchGithub = import ./fetch-github/default.nix self;
fetchGitlab = import ./fetch-gitlab/default.nix self;
fetchNixpkgs = import ./fetch-nixpkgs/default.nix self;
fetchRubyGem = import ./fetch-rubygem/default.nix self;
fetchUrl = import ./fetch-url/default.nix self;
inherit (self.__nixpkgs__.lib) filterAttrs;
inherit (self.__nixpkgs__.lib.lists) flatten;
inherit (__nixpkgs__.lib) filterAttrs;
inherit (__nixpkgs__.lib.lists) flatten;
formatBash = import ./format-bash/default.nix self;
formatNix = import ./format-nix/default.nix self;
formatPython = import ./format-python/default.nix self;
Expand All @@ -51,12 +53,12 @@
fromYaml = import ./from-yaml/default.nix self;
fromYamlFile = path: self.fromYaml (builtins.readFile path);
gitlabCi = import ./gitlab-ci/default.nix;
inherit (self.__nixpkgs__.lib.strings) hasPrefix;
inherit (self.__nixpkgs__.lib.strings) hasSuffix;
inherit (self.__nixpkgs__.stdenv) isDarwin;
inherit (self.__nixpkgs__.stdenv) isLinux;
inherit (__nixpkgs__.lib.strings) hasPrefix;
inherit (__nixpkgs__.lib.strings) hasSuffix;
inherit (__nixpkgs__.stdenv) isDarwin;
inherit (__nixpkgs__.stdenv) isLinux;
libGit = import ./lib-git/default.nix self;
listOptional = self.__nixpkgs__.lib.lists.optional;
listOptional = __nixpkgs__.lib.lists.optional;
lintClojure = import ./lint-clojure/default.nix self;
lintGitCommitMsg = import ./lint-git-commit-msg/default.nix self;
lintGitMailMap = import ./lint-git-mailmap/default.nix self;
Expand All @@ -67,7 +69,7 @@
lintTerraform = import ./lint-terraform/default.nix self;
lintWithAjv = import ./lint-with-ajv/default.nix self;
lintWithLizard = import ./lint-with-lizard/default.nix self;
inherit (self.__nixpkgs__.lib.filesystem) listFilesRecursive;
inherit (__nixpkgs__.lib.filesystem) listFilesRecursive;
makeContainerImage = import ./make-container-image/default.nix self;
makeDerivation = import ./make-derivation/default.nix self;
makeDerivationParallel = import ./make-derivation-parallel/default.nix self;
Expand All @@ -79,6 +81,7 @@
makeNodeJsVersion = import ./make-node-js-version/default.nix self;
makeNomadEnvironment = import ./make-nomad-environment/default.nix self;
makePythonPypiEnvironment = import ./make-python-pypi-environment/default.nix self;
makePythonPyprojectPackage = import ./make-python-pyproject-package/default.nix;
makePythonVersion = import ./make-python-version/default.nix self;
makeRubyGemsEnvironment = import ./make-ruby-gems-environment/default.nix self;
makeRubyGemsInstall = import ./make-ruby-gems-install/default.nix self;
Expand All @@ -99,19 +102,20 @@
makeWorkspaceForTerraformFromEnv = import ./make-workspace-for-terraform-from-env/default.nix self;
managePorts = import ./manage-ports/default.nix self;
patchShebangs = import ./patch-shebangs/default.nix self;
inherit (self.__nixpkgs__.lib) removePrefix;
pythonOverrideUtils = import ./python-override-utils/default.nix;
inherit (__nixpkgs__.lib) removePrefix;
secureKubernetesWithRbacPolice = import ./secure-kubernetes-with-rbac-police/default.nix self;
securePythonWithBandit = import ./secure-python-with-bandit/default.nix self;
sortAscii = builtins.sort (a: b: a < b);
sortAsciiCaseless = builtins.sort (a: b: self.__nixpkgs__.lib.toLower a < self.__nixpkgs__.lib.toLower b);
sortAsciiCaseless = builtins.sort (a: b: __nixpkgs__.lib.toLower a < __nixpkgs__.lib.toLower b);
stringCapitalize = import ./string-capitalize/default.nix self;
sublist = import ./sublist/default.nix self;
taintTerraform = import ./taint-terraform/default.nix self;
testLicense = import ./test-license/default.nix self;
testTerraform = import ./test-terraform/default.nix self;
testPullRequest = import ./test-pull-request/default.nix self;
testPython = import ./test-python/default.nix self;
toDerivationName = self.__nixpkgs__.lib.strings.sanitizeDerivationName;
toDerivationName = __nixpkgs__.lib.strings.sanitizeDerivationName;
toBashArray = import ./to-bash-array/default.nix self;
toBashMap = import ./to-bash-map/default.nix self;
toFileJson = import ./to-file-json/default.nix self;
Expand All @@ -120,7 +124,5 @@
toFileJsonFromFileYaml = import ./to-file-json-from-file-yaml/default.nix self;
toFileLst = import ./to-file-lst/default.nix;
});

sources = import ../nix/sources.nix;
in
assert args.isDarwin || args.isLinux; args
9 changes: 9 additions & 0 deletions src/args/make-python-pyproject-package/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
buildEnv,
buildPythonPackage,
pkgDeps,
src,
}:
import ./generic_builder {
inherit buildEnv buildPythonPackage pkgDeps src;
}
11 changes: 11 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/check.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{pkg}: let
build_check = check:
pkg.overridePythonAttrs (
old: {
checkPhase = [old."${check}"];
}
);
in {
tests = build_check "test_check";
types = build_check "type_check";
}
18 changes: 18 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
buildEnv,
buildPythonPackage,
pkgDeps,
src,
}: let
metadata = import ./metadata.nix src;
pkg = import ./pkg {
inherit buildPythonPackage metadata pkgDeps src;
};
env = import ./env.nix {
inherit buildEnv pkgDeps pkg;
};
checks = import ./check.nix {inherit pkg;};
in {
inherit pkg env;
check = checks;
}
14 changes: 14 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/env.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
buildEnv,
pkgDeps,
pkg,
}: let
build_env = extraLibs:
buildEnv {
inherit extraLibs;
ignoreCollisions = false;
};
in {
runtime = build_env [pkg];
dev = build_env (pkgDeps.runtime_deps ++ pkgDeps.test_deps);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
src: let
_metadata = (builtins.fromTOML (builtins.readFile "${src}/pyproject.toml")).project;
file_str = builtins.readFile "${src}/${_metadata.name}/__init__.py";
match = builtins.match ".*__version__ *= *\"(.+)\"\n.*" file_str;
version = builtins.elemAt match 0;
in
_metadata // {inherit version;}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# shellcheck shell=bash

echo "Executing test phase" \
&& pytest --version \
&& pytest ./tests \
&& echo "Finished test phase"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# shellcheck shell=bash

echo "Executing type check phase" \
&& mypy --version \
&& mypy . --config-file ./mypy.ini \
&& echo "Finished type check phase"
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
buildPythonPackage,
metadata,
pkgDeps,
src,
}: let
type_check = ./check/types.sh;
test_check = ./check/tests.sh;
in
buildPythonPackage {
inherit src type_check test_check;
inherit (metadata) version;
pname = metadata.name;
format = "pyproject";
checkPhase = [
''
source ${type_check} \
&& source ${test_check} \
''
];
doCheck = true;
pythonImportsCheck = [metadata.name];
nativeBuildInputs = pkgDeps.build_deps;
propagatedBuildInputs = pkgDeps.runtime_deps;
nativeCheckInputs = pkgDeps.test_deps;
}
48 changes: 48 additions & 0 deletions src/args/python-override-utils/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# python override utils,
# useful when overriding pkgs from an environment to ensure no collisions
let
recursive_python_pkg_override = is_pkg: override: let
# is_pkg: Derivation -> Bool
# override: Derivation -> Derivation
self = recursive_python_pkg_override is_pkg override;
in
pkg:
if is_pkg pkg
then override pkg
else if pkg ? overridePythonAttrs && pkg ? pname
then
pkg.overridePythonAttrs (
builtins.mapAttrs (_: value:
if builtins.isList value
then map self value
else self value)
)
else pkg;

# no_check_override: Derivation -> Derivation
no_check_override = recursive_python_pkg_override (pkg: pkg ? overridePythonAttrs && pkg ? pname) (
pkg:
pkg.overridePythonAttrs (
old:
(
builtins.mapAttrs (_: value:
if builtins.isList value
then map no_check_override value
else no_check_override value)
old
)
// {
doCheck = false;
}
)
);

# replace_pkg: List[str] -> Derivation -> Derivation
replace_pkg = names: new_pkg:
recursive_python_pkg_override (
x: x ? overridePythonAttrs && x ? pname && builtins.elem x.pname names
) (_: new_pkg);
in {
inherit recursive_python_pkg_override no_check_override replace_pkg;
compose = functions: val: builtins.foldl' (x: f: f x) val functions;
}

0 comments on commit acf54c6

Please sign in to comment.