Skip to content

Commit

Permalink
feat(back): fluidattacks#1146 make python pyproject
Browse files Browse the repository at this point in the history
- add make-python-pyproject buiding from nixpkgs functions
- add python-override-utils
  • Loading branch information
danmur97 committed Sep 5, 2023
1 parent 8ccf225 commit 0161eda
Show file tree
Hide file tree
Showing 10 changed files with 172 additions and 21 deletions.
44 changes: 23 additions & 21 deletions src/args/agnostic.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
system ? builtins.currentSystem,
}: let
fix' = __unfix__: let x = __unfix__ x // {inherit __unfix__;}; in x;

args = fix' (self: {
sources = import ../nix/sources.nix;
args = fix' (self: let
__nixpkgs__ = import sources.nixpkgs {inherit system;};
in {
inherit __nixpkgs__;
__nixpkgsSrc__ = sources.nixpkgs;
__shellCommands__ = ./shell-commands/template.sh;
__shellOptions__ = ./shell-options/template.sh;
Expand All @@ -19,27 +21,27 @@
__toModuleOutputs__ = import ./to-module-outputs/default.nix self;
asContent = import ./as-content/default.nix;
attrsGet = import ./attrs-get/default.nix;
attrsMapToList = self.__nixpkgs__.lib.mapAttrsToList;
attrsMerge = builtins.foldl' self.__nixpkgs__.lib.recursiveUpdate {};
attrsOptional = self.__nixpkgs__.lib.optionalAttrs;
attrsMapToList = __nixpkgs__.lib.mapAttrsToList;
attrsMerge = builtins.foldl' __nixpkgs__.lib.recursiveUpdate {};
attrsOptional = __nixpkgs__.lib.optionalAttrs;
calculateCvss3 = import ./calculate-cvss-3/default.nix self;
calculateScorecard = import ./calculate-scorecard/default.nix self;
chunks = import ./chunks/default.nix self;
computeOnAwsBatch = import ./compute-on-aws-batch/default.nix self;
deployContainerImage = import ./deploy-container-image/default.nix self;
deployNomad = import ./deploy-nomad/default.nix self;
deployTerraform = import ./deploy-terraform/default.nix self;
inherit (self.__nixpkgs__.lib.strings) escapeShellArg;
inherit (self.__nixpkgs__.lib.strings) escapeShellArgs;
inherit (self.__nixpkgs__.lib) fakeSha256;
inherit (__nixpkgs__.lib.strings) escapeShellArg;
inherit (__nixpkgs__.lib.strings) escapeShellArgs;
inherit (__nixpkgs__.lib) fakeSha256;
fetchArchive = import ./fetch-archive/default.nix self;
fetchGithub = import ./fetch-github/default.nix self;
fetchGitlab = import ./fetch-gitlab/default.nix self;
fetchNixpkgs = import ./fetch-nixpkgs/default.nix self;
fetchRubyGem = import ./fetch-rubygem/default.nix self;
fetchUrl = import ./fetch-url/default.nix self;
inherit (self.__nixpkgs__.lib) filterAttrs;
inherit (self.__nixpkgs__.lib.lists) flatten;
inherit (__nixpkgs__.lib) filterAttrs;
inherit (__nixpkgs__.lib.lists) flatten;
formatBash = import ./format-bash/default.nix self;
formatNix = import ./format-nix/default.nix self;
formatPython = import ./format-python/default.nix self;
Expand All @@ -51,12 +53,12 @@
fromYaml = import ./from-yaml/default.nix self;
fromYamlFile = path: self.fromYaml (builtins.readFile path);
gitlabCi = import ./gitlab-ci/default.nix;
inherit (self.__nixpkgs__.lib.strings) hasPrefix;
inherit (self.__nixpkgs__.lib.strings) hasSuffix;
inherit (self.__nixpkgs__.stdenv) isDarwin;
inherit (self.__nixpkgs__.stdenv) isLinux;
inherit (__nixpkgs__.lib.strings) hasPrefix;
inherit (__nixpkgs__.lib.strings) hasSuffix;
inherit (__nixpkgs__.stdenv) isDarwin;
inherit (__nixpkgs__.stdenv) isLinux;
libGit = import ./lib-git/default.nix self;
listOptional = self.__nixpkgs__.lib.lists.optional;
listOptional = __nixpkgs__.lib.lists.optional;
lintClojure = import ./lint-clojure/default.nix self;
lintGitCommitMsg = import ./lint-git-commit-msg/default.nix self;
lintGitMailMap = import ./lint-git-mailmap/default.nix self;
Expand All @@ -67,7 +69,7 @@
lintTerraform = import ./lint-terraform/default.nix self;
lintWithAjv = import ./lint-with-ajv/default.nix self;
lintWithLizard = import ./lint-with-lizard/default.nix self;
inherit (self.__nixpkgs__.lib.filesystem) listFilesRecursive;
inherit (__nixpkgs__.lib.filesystem) listFilesRecursive;
makeContainerImage = import ./make-container-image/default.nix self;
makeDerivation = import ./make-derivation/default.nix self;
makeDerivationParallel = import ./make-derivation-parallel/default.nix self;
Expand All @@ -79,6 +81,7 @@
makeNodeJsVersion = import ./make-node-js-version/default.nix self;
makeNomadEnvironment = import ./make-nomad-environment/default.nix self;
makePythonPypiEnvironment = import ./make-python-pypi-environment/default.nix self;
makePythonPyprojectPackage = import ./make-python-pyproject-package/default.nix self;
makePythonVersion = import ./make-python-version/default.nix self;
makeRubyGemsEnvironment = import ./make-ruby-gems-environment/default.nix self;
makeRubyGemsInstall = import ./make-ruby-gems-install/default.nix self;
Expand All @@ -99,19 +102,20 @@
makeWorkspaceForTerraformFromEnv = import ./make-workspace-for-terraform-from-env/default.nix self;
managePorts = import ./manage-ports/default.nix self;
patchShebangs = import ./patch-shebangs/default.nix self;
inherit (self.__nixpkgs__.lib) removePrefix;
pythonOverrideUtils = import ./python-override-utils/default.nix self;
inherit (__nixpkgs__.lib) removePrefix;
secureKubernetesWithRbacPolice = import ./secure-kubernetes-with-rbac-police/default.nix self;
securePythonWithBandit = import ./secure-python-with-bandit/default.nix self;
sortAscii = builtins.sort (a: b: a < b);
sortAsciiCaseless = builtins.sort (a: b: self.__nixpkgs__.lib.toLower a < self.__nixpkgs__.lib.toLower b);
sortAsciiCaseless = builtins.sort (a: b: __nixpkgs__.lib.toLower a < __nixpkgs__.lib.toLower b);
stringCapitalize = import ./string-capitalize/default.nix self;
sublist = import ./sublist/default.nix self;
taintTerraform = import ./taint-terraform/default.nix self;
testLicense = import ./test-license/default.nix self;
testTerraform = import ./test-terraform/default.nix self;
testPullRequest = import ./test-pull-request/default.nix self;
testPython = import ./test-python/default.nix self;
toDerivationName = self.__nixpkgs__.lib.strings.sanitizeDerivationName;
toDerivationName = __nixpkgs__.lib.strings.sanitizeDerivationName;
toBashArray = import ./to-bash-array/default.nix self;
toBashMap = import ./to-bash-map/default.nix self;
toFileJson = import ./to-file-json/default.nix self;
Expand All @@ -120,7 +124,5 @@
toFileJsonFromFileYaml = import ./to-file-json-from-file-yaml/default.nix self;
toFileLst = import ./to-file-lst/default.nix;
});

sources = import ../nix/sources.nix;
in
assert args.isDarwin || args.isLinux; args
13 changes: 13 additions & 0 deletions src/args/make-python-pyproject-package/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
buildEnv,
buildPythonPackage,
nixpkgs,
src,
}:
# pkgDeps: AttrsOf [
# keys: Literal["runtime_deps", "build_deps", "test_deps"]
# values: List[Derivation]
# ]
import ./generic_builder {
inherit buildEnv buildPythonPackage pkgDeps src;
}
11 changes: 11 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/check.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{pkg}: let
build_check = check:
pkg.overridePythonAttrs (
old: {
checkPhase = [old."${check}"];
}
);
in {
tests = build_check "test_check";
types = build_check "type_check";
}
18 changes: 18 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
buildEnv,
buildPythonPackage,
pkgDeps,
src,
}: let
metadata = import ./metadata.nix src;
pkg = import ./pkg {
inherit buildPythonPackage metadata pkgDeps src;
};
env = import ./env.nix {
inherit buildEnv pkgDeps pkg;
};
checks = import ./check.nix {inherit pkg;};
in {
inherit pkg env;
check = checks;
}
14 changes: 14 additions & 0 deletions src/args/make-python-pyproject-package/generic_builder/env.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
buildEnv,
pkgDeps,
pkg,
}: let
build_env = extraLibs:
buildEnv {
inherit extraLibs;
ignoreCollisions = false;
};
in {
runtime = build_env [pkg];
dev = build_env (pkgDeps.runtime_deps ++ pkgDeps.test_deps);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
src: let
_metadata = (builtins.fromTOML (builtins.readFile "${src}/pyproject.toml")).project;
file_str = builtins.readFile "${src}/${_metadata.name}/__init__.py";
match = builtins.match ".*__version__ *= *\"(.+)\"\n.*" file_str;
version = builtins.elemAt match 0;
in
_metadata // {inherit version;}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# shellcheck shell=bash

echo "Executing test phase" \
&& pytest --version \
&& pytest ./tests \
&& echo "Finished test phase"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# shellcheck shell=bash

echo "Executing type check phase" \
&& mypy --version \
&& mypy . --config-file ./mypy.ini \
&& echo "Finished type check phase"
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
buildPythonPackage,
metadata,
pkgDeps,
src,
}: let
type_check = ./check/types.sh;
test_check = ./check/tests.sh;
in
buildPythonPackage {
inherit src type_check test_check;
inherit (metadata) version;
pname = metadata.name;
format = "pyproject";
checkPhase = [
''
source ${type_check} \
&& source ${test_check} \
''
];
doCheck = true;
pythonImportsCheck = [metadata.name];
nativeBuildInputs = pkgDeps.build_deps;
propagatedBuildInputs = pkgDeps.runtime_deps;
nativeCheckInputs = pkgDeps.test_deps;
}
48 changes: 48 additions & 0 deletions src/args/python-override-utils/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# python override utils,
# useful when overriding pkgs from an environment to ensure no collisions
let
recursive_python_pkg_override = is_pkg: override: let
# is_pkg: Derivation -> Bool
# override: Derivation -> Derivation
self = recursive_python_pkg_override is_pkg override;
in
pkg:
if is_pkg pkg
then override pkg
else if pkg ? overridePythonAttrs && pkg ? pname
then
pkg.overridePythonAttrs (
builtins.mapAttrs (_: value:
if builtins.isList value
then map self value
else self value)
)
else pkg;

# no_check_override: Derivation -> Derivation
no_check_override = recursive_python_pkg_override (pkg: pkg ? overridePythonAttrs && pkg ? pname) (
pkg:
pkg.overridePythonAttrs (
old:
(
builtins.mapAttrs (_: value:
if builtins.isList value
then map no_check_override value
else no_check_override value)
old
)
// {
doCheck = false;
}
)
);

# replace_pkg: List[str] -> Derivation -> Derivation
replace_pkg = names: new_pkg:
recursive_python_pkg_override (
x: x ? overridePythonAttrs && x ? pname && builtins.elem x.pname names
) (_: new_pkg);
in {
inherit recursive_python_pkg_override no_check_override replace_pkg;
compose = functions: val: builtins.foldl' (x: f: f x) val functions;
}

0 comments on commit 0161eda

Please sign in to comment.