From 0a513df838207666dd442a6e0a5676260788bd0d Mon Sep 17 00:00:00 2001 From: David Arnold Date: Thu, 15 Jun 2023 17:13:50 -0500 Subject: [PATCH] feat: start using lazyDerivation for faster TUI response times --- cells/lib/ops.nix | 1 + cells/lib/ops/lazyDerivation.nix | 108 ++++++++++++++++++++++++++++++ cells/lib/ops/mkDevOCI.nix | 2 +- cells/lib/ops/mkOCI.nix | 110 +++++++++++++++++-------------- cells/lib/ops/mkOperable.nix | 27 ++++---- cells/lib/ops/mkStandardOCI.nix | 19 ++---- cells/lib/ops/writeScript.nix | 75 ++++++++++----------- deprecation.nix | 9 --- flake.lock | 6 +- src/blocktypes/containers.nix | 18 +++-- 10 files changed, 242 insertions(+), 133 deletions(-) create mode 100644 cells/lib/ops/lazyDerivation.nix diff --git a/cells/lib/ops.nix b/cells/lib/ops.nix index 80e006bb..42141cf1 100644 --- a/cells/lib/ops.nix +++ b/cells/lib/ops.nix @@ -14,6 +14,7 @@ in { mkSetup = import ./ops/mkSetup.nix {inherit inputs cell;}; mkUser = import ./ops/mkUser.nix {inherit inputs cell;}; writeScript = import ./ops/writeScript.nix {inherit inputs cell;}; + lazyDerivation = import ./ops/lazyDerivation.nix {inherit inputs cell;}; mkOCI = import ./ops/mkOCI.nix {inherit inputs cell;}; mkDevOCI = import ./ops/mkDevOCI.nix {inherit inputs cell;}; diff --git a/cells/lib/ops/lazyDerivation.nix b/cells/lib/ops/lazyDerivation.nix new file mode 100644 index 00000000..8e09ce5d --- /dev/null +++ b/cells/lib/ops/lazyDerivation.nix @@ -0,0 +1,108 @@ +{ + inputs, + cell, +}: let + inherit (inputs) nixpkgs std; + l = nixpkgs.lib // builtins; + inherit (l) throwIfNot; +in ( + if l ? lazyDerivation + then l.lazyDerivation + else + { + /* + Restrict a derivation to a predictable set of attribute names, so + that the returned attrset is not strict in the actual derivation, + saving a lot of computation when the derivation is non-trivial. + + This is useful in situations where a derivation might only be used for its + passthru attributes, improving evaluation performance. + + The returned attribute set is lazy in `derivation`. Specifically, this + means that the derivation will not be evaluated in at least the + situations below. + + For illustration and/or testing, we define derivation such that its + evaluation is very noticeable. + + let derivation = throw "This won't be evaluated."; + + In the following expressions, `derivation` will _not_ be evaluated: + + (lazyDerivation { inherit derivation; }).type + + attrNames (lazyDerivation { inherit derivation; }) + + (lazyDerivation { inherit derivation; } // { foo = true; }).foo + + (lazyDerivation { inherit derivation; meta.foo = true; }).meta + + In these expressions, it `derivation` _will_ be evaluated: + + "${lazyDerivation { inherit derivation }}" + + (lazyDerivation { inherit derivation }).outPath + + (lazyDerivation { inherit derivation }).meta + + And the following expressions are not valid, because the refer to + implementation details and/or attributes that may not be present on + some derivations: + + (lazyDerivation { inherit derivation }).buildInputs + + (lazyDerivation { inherit derivation }).passthru + + (lazyDerivation { inherit derivation }).pythonPath + + */ + lazyDerivation = args @ { + # The derivation to be wrapped. + derivation, + # Optional meta attribute. + # + # While this function is primarily about derivations, it can improve + # the `meta` package attribute, which is usually specified through + # `mkDerivation`. + meta ? null, + # Optional extra values to add to the returned attrset. + # + # This can be used for adding package attributes, such as `tests`. + passthru ? {}, + }: let + # These checks are strict in `drv` and some `drv` attributes, but the + # attrset spine returned by lazyDerivation does not depend on it. + # Instead, the individual derivation attributes do depend on it. + checked = + throwIfNot (derivation.type or null == "derivation") + "lazySimpleDerivation: input must be a derivation." + throwIfNot + (derivation.outputs == ["out"]) + # Supporting multiple outputs should be a matter of inheriting more attrs. + "The derivation ${derivation.name or ""} has multiple outputs. This is not supported by lazySimpleDerivation yet. Support could be added, and be useful as long as the set of outputs is known in advance, without evaluating the actual derivation." + derivation; + in + { + # Hardcoded `type` + # + # `lazyDerivation` requires its `derivation` argument to be a derivation, + # so if it is not, that is a programming error by the caller and not + # something that `lazyDerivation` consumers should be able to correct + # for after the fact. + # So, to improve laziness, we assume correctness here and check it only + # when actual derivation values are accessed later. + type = "derivation"; + + # A fixed set of derivation values, so that `lazyDerivation` can return + # its attrset before evaluating `derivation`. + # This must only list attributes that are available on _all_ derivations. + inherit (checked) outputs out outPath outputName drvPath name system; + + # The meta attribute can either be taken from the derivation, or if the + # `lazyDerivation` caller knew a shortcut, be taken from there. + meta = args.meta or checked.meta; + } + // passthru; + } + .lazyDerivation +) diff --git a/cells/lib/ops/mkDevOCI.nix b/cells/lib/ops/mkDevOCI.nix index 4c7ec092..331264e0 100644 --- a/cells/lib/ops/mkDevOCI.nix +++ b/cells/lib/ops/mkDevOCI.nix @@ -42,7 +42,7 @@ in vscode ? false, slim ? false, user ? "user", - tag ? "", + tag ? null, pkgs ? [], setup ? [], perms ? [], diff --git a/cells/lib/ops/mkOCI.nix b/cells/lib/ops/mkOCI.nix index dc87a55a..cc9be053 100644 --- a/cells/lib/ops/mkOCI.nix +++ b/cells/lib/ops/mkOCI.nix @@ -31,7 +31,7 @@ in tag ? if meta ? tags && (l.length meta.tags) > 0 then l.head meta.tags - else "", + else null, setup ? [], layers ? [], runtimeInputs ? [], @@ -47,57 +47,69 @@ in mkdir -p $out/bin ln -s ${l.getExe entrypoint} $out/bin/entrypoint ''; - options' = - { - inherit name meta; - # Layers are nested to reduce duplicate paths in the image - layers = - [ - # Primary layer is the entrypoint layer - (n2c.buildLayer { - deps = [entrypoint]; - maxLayers = 50; - layers = [ - # Runtime inputs layer - (n2c.buildLayer { - deps = runtimeInputs; - maxLayers = 10; - }) - ]; - }) - ] - ++ layers; + image = + l.throwIf (args ? tag && meta ? tags) + "mkOCI/mkStandardOCI/mkDevOCI: use of `tag` and `meta.tags` arguments are not supported together. Remove the former." + n2c.buildImage ( + l.recursiveUpdate options { + inherit name tag; - maxLayers = 25; - copyToRoot = - [ - (nixpkgs.buildEnv { - name = "root"; - paths = [setupLinks] ++ setup; - }) - ] - ++ options.copyToRoot or []; + # Layers are nested to reduce duplicate paths in the image + layers = + [ + # Primary layer is the entrypoint layer + (n2c.buildLayer { + deps = [entrypoint]; + maxLayers = 50; + layers = [ + # Runtime inputs layer + (n2c.buildLayer { + deps = runtimeInputs; + maxLayers = 10; + }) + ]; + }) + ] + ++ layers; - config = l.recursiveUpdate config { - User = uid; - Group = gid; - Entrypoint = ["/bin/entrypoint"]; - Labels = l.mapAttrs' (n: v: l.nameValuePair "org.opencontainers.image.${n}" v) labels; - }; + maxLayers = 25; + copyToRoot = + [ + (nixpkgs.buildEnv { + name = "root"; + paths = [setupLinks] ++ setup; + }) + ] + ++ options.copyToRoot or []; - # Setup tasks can include permissions via the passthru.perms attribute - perms = l.flatten ((l.map (s: l.optionalAttrs (s ? passthru && s.passthru ? perms) s.passthru.perms)) setup) ++ perms; - } - // l.throwIf (args ? tag && meta ? tags) - "mkOCI: use of `tag` and `meta.tags` arguments are not supported together. Remove the former." - (l.optionalAttrs (tag != "") {inherit tag;}); + config = l.recursiveUpdate config { + User = uid; + Group = gid; + Entrypoint = ["/bin/entrypoint"]; + Labels = l.mapAttrs' (n: v: l.nameValuePair "org.opencontainers.image.${n}" v) labels; + }; - image = n2c.buildImage (l.recursiveUpdate options options'); + # Setup tasks can include permissions via the passthru.perms attribute + perms = l.flatten ((l.map (s: l.optionalAttrs (s ? passthru && s.passthru ? perms) s.passthru.perms)) setup) ++ perms; + } + ); + in let + mainTag = + if tag != null && tag != "" + then tag + else + builtins.unsafeDiscardStringContext + (l.head (l.strings.splitString "-" (baseNameOf image.outPath))); + tags = l.unique ([mainTag] ++ meta.tags or []); in - l.removeAttrs image [ - "copyTo" - "copyToDockerDaemon" - "copyToPodman" - "copyToRegistry" - ] + cell.ops.lazyDerivation { + inherit meta; + derivation = image; + passthru = { + image.name = "${name}:${mainTag}"; + image.repo = name; + image.tag = mainTag; + image.tags = l.unique ([mainTag] ++ meta.tags or []); + }; + } diff --git a/cells/lib/ops/mkOperable.nix b/cells/lib/ops/mkOperable.nix index 98bc9be5..60c26ef8 100644 --- a/cells/lib/ops/mkOperable.nix +++ b/cells/lib/ops/mkOperable.nix @@ -30,6 +30,7 @@ in debugInputs ? [], livenessProbe ? null, readinessProbe ? null, + meta ? {}, }: let # nixpkgs.runtimeShell is a path to the shell, not a derivation runtimeShellBin = @@ -69,18 +70,20 @@ in ''; }; in - (cell.ops.writeScript - ({ - inherit runtimeInputs runtimeEnv; - name = "operable-${package.name}"; - text = '' - ${runtimeScript} - ''; - } - // l.optionalAttrs (runtimeShell != null) { - inherit runtimeShell; - })) - // { + cell.ops.lazyDerivation { + inherit meta; + derivation = + cell.ops.writeScript + ({ + inherit runtimeInputs runtimeEnv; + name = "operable-${package.name}"; + text = '' + ${runtimeScript} + ''; + } + // l.optionalAttrs (runtimeShell != null) { + inherit runtimeShell; + }); passthru = # These attributes are useful for informing later stages { diff --git a/cells/lib/ops/mkStandardOCI.nix b/cells/lib/ops/mkStandardOCI.nix index 86f9ed90..529416d8 100644 --- a/cells/lib/ops/mkStandardOCI.nix +++ b/cells/lib/ops/mkStandardOCI.nix @@ -29,7 +29,7 @@ in args @ { name, operable, - tag ? "", + tag ? null, setup ? [], uid ? "65534", gid ? "65534", @@ -40,11 +40,10 @@ in options ? {}, meta ? {}, }: let - inherit (operable) passthru; - inherit (operable.passthru) livenessProbe readinessProbe runtimeInputs runtime debug; + inherit (operable) livenessProbe readinessProbe runtimeInputs runtime debug; - hasLivenessProbe = passthru ? livenessProbe; - hasReadinessProbe = passthru ? readinessProbe; + hasLivenessProbe = operable ? livenessProbe; + hasReadinessProbe = operable ? readinessProbe; hasDebug = args.debug or false; # Link useful paths into the container. @@ -105,19 +104,15 @@ in nss = nixpkgs.writeTextDir "etc/nsswitch.conf" '' hosts: files dns ''; - - tag' = - l.throwIf (args ? tag && meta ? tags) - "mkStandardOCI: use of `tag` and `meta.tags` arguments are not supported together. Remove the former." - l.optionalString (tag != "") ((import "${inputs.self}/deprecation.nix" inputs).warnLegacyTag tag); in with dmerge; + l.throwIf (args ? tag && meta ? tags) + "mkStandardOCI: use of `tag` and `meta.tags` arguments are not supported together. Remove the former." cell.ops.mkOCI ( merge { - inherit name uid gid labels options perms config meta setup runtimeInputs; + inherit name tag uid gid labels options perms config meta setup runtimeInputs; entrypoint = operable'; - tag = tag'; # ideally '' } { # mkStandardOCI differentiators over mkOCI diff --git a/cells/lib/ops/writeScript.nix b/cells/lib/ops/writeScript.nix index 7a973709..b82c283f 100644 --- a/cells/lib/ops/writeScript.nix +++ b/cells/lib/ops/writeScript.nix @@ -18,44 +18,45 @@ in then (l.getExe runtimeShell) else runtimeShell; in - nixpkgs.writeTextFile { - inherit name; - executable = true; - destination = "/bin/${name}"; - text = - '' - #!${runtimeShell'} - # shellcheck shell=bash - set -o errexit - set -o pipefail - set -o nounset - set -o functrace - set -o errtrace - set -o monitor - set -o posix - shopt -s dotglob - - '' - + l.optionalString (runtimeInputs != []) '' - export PATH="${l.makeBinPath runtimeInputs}:$PATH" - '' - + l.optionalString (runtimeEnv != {}) '' - ${l.concatStringsSep "\n" (l.mapAttrsToList (n: v: "export ${n}=${''"$''}{${n}:-${toString v}}${''"''}") runtimeEnv)} - '' - + '' + cell.ops.lazyDerivation { + meta.mainProgram = name; + derivation = nixpkgs.writeTextFile { + inherit name; + executable = true; + destination = "/bin/${name}"; + text = + '' + #!${runtimeShell'} + # shellcheck shell=bash + set -o errexit + set -o pipefail + set -o nounset + set -o functrace + set -o errtrace + set -o monitor + set -o posix + shopt -s dotglob - ${text} - ''; + '' + + l.optionalString (runtimeInputs != []) '' + export PATH="${l.makeBinPath runtimeInputs}:$PATH" + '' + + l.optionalString (runtimeEnv != {}) '' + ${l.concatStringsSep "\n" (l.mapAttrsToList (n: v: "export ${n}=${''"$''}{${n}:-${toString v}}${''"''}") runtimeEnv)} + '' + + '' - checkPhase = - if checkPhase == null - then '' - runHook preCheck - ${nixpkgs.stdenv.shellDryRun} "$target" - ${nixpkgs.shellcheck}/bin/shellcheck "$target" - runHook postCheck - '' - else checkPhase; + ${text} + ''; - meta.mainProgram = name; + checkPhase = + if checkPhase == null + then '' + runHook preCheck + ${nixpkgs.stdenv.shellDryRun} "$target" + ${nixpkgs.shellcheck}/bin/shellcheck "$target" + runHook postCheck + '' + else checkPhase; + }; } diff --git a/deprecation.nix b/deprecation.nix index 6043b4d1..9030b25e 100644 --- a/deprecation.nix +++ b/deprecation.nix @@ -37,13 +37,4 @@ in { with `inputs.std.lib.cfg` ''; - warnLegacyTag = removeBy "July 2023" '' - The legacy upstream nix2container tag interface is deprecated, - std.lib.ops.mkStandardOCI now takes a list of tags via `meta`. - - Replace `tag` input of `mkStandardOCI` function - mkStandardOCI {tag = "foo";/* ... */} - with - mkStandardOCI {meta.tags = ["foo"]; /* ... */} - ''; } diff --git a/flake.lock b/flake.lock index 4e0b08b9..438aef22 100644 --- a/flake.lock +++ b/flake.lock @@ -203,11 +203,11 @@ ] }, "locked": { - "lastModified": 1677330646, - "narHash": "sha256-hUYCwJneMjnxTvj30Fjow6UMJUITqHlpUGpXMPXUJsU=", + "lastModified": 1685771919, + "narHash": "sha256-3lVKWrhNXjHJB6QkZ2SJaOs4X/mmYXtY6ovPVpDMOHc=", "owner": "nlewo", "repo": "nix2container", - "rev": "ebca8f58d450cae1a19c07701a5a8ae40afc9efc", + "rev": "95e2220911874064b5d809f8d35f7835184c4ddf", "type": "github" }, "original": { diff --git a/src/blocktypes/containers.nix b/src/blocktypes/containers.nix index 3e9f6b07..06ea64fb 100644 --- a/src/blocktypes/containers.nix +++ b/src/blocktypes/containers.nix @@ -24,10 +24,8 @@ target, }: let inherit (n2c.packages.${currentSystem}) skopeo-nix2container; - img = builtins.unsafeDiscardStringContext target.imageName; - tags = target.meta.tags or [(builtins.unsafeDiscardStringContext target.imageTag)]; tags' = - builtins.toFile "${target.name}-tags.json" (builtins.unsafeDiscardStringContext (builtins.concatStringsSep "\n" tags)); + builtins.toFile "${target.name}-tags.json" (builtins.concatStringsSep "\n" target.image.tags); copyFn = let skopeo = "skopeo --insecure-policy"; in '' @@ -55,11 +53,11 @@ (sharedActions.build currentSystem target) (mkCommand currentSystem { name = "print-image"; - description = "print out the image name with all tags"; + description = "print out the image.repo with all tags"; command = '' echo for tag in $(<${tags'}); do - echo "${img}:$tag" + echo "${target.image.repo}:$tag" done ''; }) @@ -68,9 +66,9 @@ description = "copy the image to its remote registry"; command = '' ${copyFn} - copy docker://${img} + copy docker://${target.image.repo} ''; - meta.images = map (tag: "${img}:${tag}") tags; + meta.image = target.image.name; proviso = let filter = ./container-publish-filter.jq; in @@ -88,7 +86,7 @@ --from-file "${filter}" \ --arg available "$( parallel -j0 scopeo_inspect ::: "$( - command jq --raw-output 'map(.meta.images[0]|strings)[]' <<< "$1" + command jq --raw-output 'map(.meta.image|strings)[]' <<< "$1" )" )" <<< "$1" @@ -103,11 +101,11 @@ ${copyFn} if command -v podman &> /dev/null; then echo "Podman detected: copy to local podman" - copy containers-storage:${img} "$@" + copy containers-storage:${target.image.repo} "$@" fi if command -v docker &> /dev/null; then echo "Docker detected: copy to local docker" - copy docker-daemon:${img} "$@" + copy docker-daemon:${target.image.repo} "$@" fi ''; })