diff --git a/lib/customisation.nix b/lib/customisation.nix index be6585d5fa6e4..9600f1f9d00b4 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -301,7 +301,26 @@ rec { in if missingArgs == { } then - makeOverridable f allArgs + if fargs ? mkPackage then + if fargs != { mkPackage = false; } then + # If your file is e.g. an attrset with a package and something else, + # use `{ callPackage }: { foo = callPackage ({ mkPackage}: ...); ... }` + # Explaining that in the error message would be too verbose and confusing. + # TODO: link to docs + throw '' + callPackage: A package function that uses `mkPackage` must only have `{ mkPackage }:` as its arguments. + Any other dependencies should be taken from the `mkPackage` callback. + Otherwise, such dependencies will not be overridable. + + To illustrate, a dependency `foo` can be retrieved with: + { mkPackage }: mkPackage ({ layers, foo }: [ + ... + ]) + '' + else + f allArgs + else + makeOverridable f allArgs # This needs to be an abort so it can't be caught with `builtins.tryEval`, # which is used by nix-env and ofborg to filter out packages that don't evaluate. # This way we're forced to fix such errors in Nixpkgs, @@ -446,9 +465,12 @@ rec { commonAttrs = { - inherit (drv) name system meta; + inherit (drv) name meta; inherit outputs; } + // optionalAttrs (drv ? system) { + inherit (drv) system; + } // optionalAttrs (drv._hydraAggregate or false) { _hydraAggregate = true; constituents = map hydraJob (flatten drv.constituents); diff --git a/lib/default.nix b/lib/default.nix index 849d6ce8f9d6a..916aeb7354813 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -80,7 +80,7 @@ let genericClosure readFile; inherit (self.fixedPoints) fix fix' converge extends composeExtensions composeManyExtensions makeExtensible makeExtensibleWithCustomName - toExtension; + encapsulate toExtension; inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath getAttrFromPath attrVals attrNames attrValues getAttrs catAttrs filterAttrs filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix index 1de5351d95aac..188ef72d26d62 100644 --- a/lib/fixed-points.nix +++ b/lib/fixed-points.nix @@ -451,6 +451,96 @@ rec { } ); + /* + Creates an overridable attrset with encapsulation. + + This is like `makeExtensible`, but only the `public` attribute of the fixed + point is returned. + + Synopsis: + + r = encapsulate (final@{extend, ...}: { + + # ... private attributes for `final` ... + + public = { + # ... returned attributes for r, in terms of `final` ... + inherit extend; # optional, don't invoke too often; see below + }; + }) + + s = r.extend (final: previous: { + + # ... updates to private attributes ... + + # optionally + public = previous.public // { + # ... updates to public attributes ... + }; + }) + + = Performance + + The `extend` function evaluates the whole fixed point all over, reusing + no "intermediate results" from the existing object. + This is necessary, because `final` has changed. + So the cost is quadratic; O(n^2) where n = number of chained invocations. + This has consequences for interface design. + Although enticing, `extend` is not suitable for directly implementing "fluent interfaces", where the caller makes many calls to `extend` via domain-specific "setters" or `with*` functions. + Fluent interfaces can not be implemented efficiently in Nix and have very little to offer over attribute sets in terms of usability.* + + Example: + + # cd nixpkgs; nix repl lib + + nix-repl> multiplier = encapsulate (self: { + a = 1; + b = 1; + public = { + r = self.a * self.b; + + # Publishing extend makes the attrset open for any kind of change. + inherit (self) extend; + + # Instead, or additionally, you can add domain-specific functions. + # Offer a single method with multiple arguments, and not a + # "fluent interface" of a method per argument, because all extension + # functions are called for every `extend`. See the Performance section. + withParams = args@{ a ? null, b ? null }: # NB: defaults are not used + self.extend (self: super: args); + + }; + }) + + nix-repl> multiplier + { extend = «lambda»; r = 1; withParams =«lambda»; } + + nix-repl> multiplier.withParams { a = 42; b = 10; } + { extend = «lambda»; r = 420; withParams =«lambda»; } + + nix-repl> multiplier3 = multiplier.extend (self: super: { + c = 1; + public = super.public // { + r = super.public.r * self.c; + }; + }) + + nix-repl> multiplier3.extend (self: super: { a = 2; b = 3; c = 10; }) + { extend = «lambda»; r = 60; withParams =«lambda»; } + + (*) Final note on Fluent APIs: While the asymptotic complexity can be fixed + by avoiding overlay extension or perhaps using it only at the end of the + chain only, one problem remains. Every method invocation has to produce + a new, immutable state value, which means copying the whole state up to + that point. + */ + encapsulate = + layerZero: + let + fixed = layerZero ({ extend = f: encapsulate (extends f layerZero); } // fixed); + in + fixed.public; + /** Convert to an extending function (overlay). diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 683c8255ca056..8e742b87623b2 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -27,6 +27,14 @@ let }; evalMinimalConfig = module: nixosLib.evalModules { modules = [ module ]; }; + # TODO: Base this on `devShell` attribute provided by the package, instead + # of prying the internals. + # Context: https://github.com/NixOS/nix/issues/7501 + getGuts = drvPkg: drvPkg // { + inherit (drvPkg.internals) drvAttrs; + outputs = drvPkg.internals.setup.outputs or [ "out" ]; + }; + nginxArguments = let nginxPort = "80"; @@ -673,7 +681,7 @@ rec { fakeRootCommands = '' mkdir -p ./home/alice chown 1000 ./home/alice - ln -s ${ + ln -s ${getGuts ( pkgs.hello.overrideAttrs ( finalAttrs: prevAttrs: { # A unique `hello` to make sure that it isn't included via another mechanism by accident. @@ -687,7 +695,7 @@ rec { }; } ) - } ./hello + )} ./hello ''; }; @@ -869,7 +877,7 @@ rec { nix-shell-basic = streamNixShellImage { name = "nix-shell-basic"; tag = "latest"; - drv = pkgs.hello; + drv = getGuts pkgs.hello; }; nix-shell-hook = streamNixShellImage { @@ -972,7 +980,7 @@ rec { nix-shell-build-derivation = streamNixShellImage { name = "nix-shell-build-derivation"; tag = "latest"; - drv = pkgs.hello; + drv = getGuts pkgs.hello; run = '' buildDerivation $out/bin/hello diff --git a/pkgs/build-support/node/build-npm-package/layer.nix b/pkgs/build-support/node/build-npm-package/layer.nix new file mode 100644 index 0000000000000..582ec6cebe877 --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/layer.nix @@ -0,0 +1,75 @@ +/** + `mkPackage` layer for building NPM packages. +*/ +{ + lib, + stdenv, + fetchNpmDeps, + buildPackages, + nodejs, + cctools, +}: + +let + # The fetcher needs the unpack related attributes + fetchInherited = { + src = null; + srcs = null; + sourceRoot = null; + prePatch = null; + patches = null; + postPatch = null; + patchFlags = null; + }; +in +this: old: +let + inherit (this) deps name version; +in +{ + deps = { + inherit (deps.nodejs) fetchNpmDeps; + npmHooks = buildPackages.npmHooks.override { + inherit (deps) nodejs; + }; + inherit (deps.npmHooks) npmConfigHook npmBuildHook npmInstallHook; + } // old.deps; + /** + Arguments for fetchNpmDeps + */ + npmFetch = + { + name = "${name}-${version}-npm-deps"; + hash = throw "Please specify npmFetch.hash in the package definition of ${name}"; + forceEmptyCache = false; + forceGitDeps = false; + patchFlags = ""; + postPatch = ""; + prePatch = ""; + } + // builtins.intersectAttrs fetchInherited this.setup + // old.npmFetch; + setup = old.setup or { } // { + inherit (deps) nodejs; + npmDeps = fetchNpmDeps this.npmFetch; + npmPruneFlags = old.setup.npmPruneFlags or this.setup.npmInstallFlags or [ ]; + npmBuildScript = "build"; + nativeBuildInputs = + old.setup.nativeBuildInputs + ++ [ + deps.nodejs + deps.npmConfigHook + deps.npmBuildHook + deps.npmInstallHook + deps.nodejs.python + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ cctools ]; + buildInputs = old.setup.buildInputs or [ ] ++ [ deps.nodejs ]; + strictDeps = true; + # Stripping takes way too long with the amount of files required by a typical Node.js project. + dontStrip = old.dontStrip or true; + }; + meta = (old.meta or { }) // { + platforms = old.meta.platforms or deps.nodejs.meta.platforms; + }; +} diff --git a/pkgs/build-support/package/make-package.nix b/pkgs/build-support/package/make-package.nix new file mode 100644 index 0000000000000..81052f7675781 --- /dev/null +++ b/pkgs/build-support/package/make-package.nix @@ -0,0 +1,259 @@ +{ + lib, + callPackage, + stdenv, + config, + ... +}: +let + checkMeta = import ../../stdenv/generic/check-meta.nix { + inherit lib config; + # Nix itself uses the `system` field of a derivation to decide where + # to build it. This is a bit confusing for cross compilation. + inherit (stdenv) hostPlatform; + }; + + baseLayer = this: { + pos = builtins.unsafeGetAttrPos "name" this; + # TODO: drvAttrs won't be available in RFC 92 dynamic derivations or multi-derivation packages. + validity = checkMeta.assertValidity { + inherit (this.public) meta; + attrs = this.drvAttrs // { + inherit (this) meta; + }; + }; + + # In the repl, use :p pkg.help + # Since https://github.com/NixOS/nix/pull/10208 + help = + if this.deps == { } then + '' + # Overriding dependencies + + This package does not have overridable dependencies using the .override attribute. + + '' + else + '' + # Overriding dependencies + + This package allows its dependencies to be overridden, using the .override + attribute. For example: + + pkg.override (old: { dep = f old.dep; }) + + Instead of dep, you may set the following attributes: + + ${lib.concatMapStringsSep ", " lib.strings.escapeNixIdentifier (lib.attrNames this.deps)} + ''; + + deps = { }; + + public = + builtins.intersectAttrs { + tests = null; + version = null; + } this + // { + /* + The marker for a [package attribute set](https://nixos.org/manual/nix/stable/glossary.html#package-attribute-set). + The value is "derivation" for historical reasons. + */ + type = "derivation"; + + # FIXME: this assumes this.drvAttrs, which is a bad dependency on the + # derivation layer + meta = checkMeta.commonMeta { + inherit (this) validity pos; + attrs = this.drvAttrs // { + inherit (this) meta; + }; + references = + this.drvAttrs.nativeBuildInputs or [ ] + ++ this.drvAttrs.buildInputs or [ ] + ++ this.drvAttrs.propagatedNativeBuildInputs or [ ] + ++ this.drvAttrs.propagatedBuildInputs or [ ]; + }; + + inherit (this) help name; + + override = + f: + this.extend ( + this: old: { + deps = old.deps // f old.deps; + } + ); + + internals = this; + + # TODO: Support legacy attrs like passthru? + overrideAttrs = + f: + this.extend ( + this: old: { + setup = + old.setup + // (lib.toExtension f) ( + this.setup or { } + // { + finalPackage = this.public; + } + ) old.setup or { }; + } + ); + }; + }; + + layers.derivation = + { stdenv, ... }: + this: old: + let + outputs = lib.genAttrs (this.drvAttrs.outputs) ( + outputName: + this.public + // { + outPath = + assert this.validity.handled; + this.drvOutAttrs.${outputName}; + inherit outputName; + outputSpecified = true; + } + ); + in + { + drvAttrs = stdenv.makeDerivationArgument ( + ( + if this ? version then + { + pname = this.name; + inherit (this) version; + } + else + { + inherit (this) name; + } + ) + // this.setup + ); + drvOutAttrs = builtins.derivationStrict this.drvAttrs; + public = + old.public + // rec { + outPath = + assert this.validity.handled; + this.drvOutAttrs.${outputName}; + outputName = lib.head this.drvAttrs.outputs; + # legacy attribute for single-drv packages + drvPath = + assert this.validity.handled; + this.drvOutAttrs.drvPath; + } + // outputs; + }; + + layers.withDeps = + f: externalArgs: this: old: + let + fargs = lib.functionArgs f; + spy = lib.setFunctionArgs (args: { inherit args; }) ( + lib.mapAttrs ( + _k: _v: + # say we have a default, so that unknown attrs aren't a problem and + # they can be defined by a subsequent layer. + true + ) fargs + ); + # TODO: make callPackage a parameter? + values = (callPackage spy externalArgs).args; + old2 = old // { + deps = old.deps or { } // values; + inherit values; + }; + r = f ( + lib.mapAttrs ( + name: hasDefault: + builtins.addErrorContext "while evaluating the package function argument `${name}`" ( + externalArgs.${name} or (this.deps.${name} + or (throw "Dependency ${name} went missing from the package internal `deps` attribute. Did you forget to preserve previous deps? Write e.g. `deps = prev.deps // { ... }`") + ) + ) + ) fargs + ); + r' = if lib.isList r then lib.composeManyExtensions r else r; + in + old2 // r' this old2; + # lib.composeExtensions f ({ }) + + # TODO: layers.meson + # TODO: layers.cmake + # TODO: layers.pkg-config + # TODO: layers. + + layers.noop = _this: _old: { }; + + layers.buildNpmPackage = callPackage ../node/build-npm-package/layer.nix { }; + + mkPackageWith = + { + # these are not overridable by the layer implementations - not suited for `deps` + externalDeps ? { inherit layers; }, + }: + f: + lib.encapsulate ( + this: + let + baseLayer' = + x: + baseLayer x + // { + /** + Extend the package layers with the given function. + */ + extend = + f: + this.extend ( + this: old: { + userLayer = lib.composeExtensions old.userLayer f; + } + ); + }; + in + # The root of the mkPackage fixpoint is responsible for managing the deps, + # and combining the layers (without adding an extra fixpoint). + # Virtually all package logic happens in userLayer. + { + userLayer = + final: prev: + this.externalDeps.layers.withDeps f ( + this.externalDeps + // { + # Package attributes + inherit final prev; + # Dependencies + /** + Override the package dependencies that are not overridable by the individual layer implementations, + Notably, the `layers` attribute. + */ + overrideExternalDeps = + newDeps: + this.extend ( + this: old: { + externalDeps = old.externalDeps // newDeps; + } + ); + } + ) final prev; + inherit externalDeps; + package = lib.extends this.userLayer baseLayer' this.package; + inherit (this.package) public; + } + ); + mkPackage = mkPackageWith { }; +in +{ + inherit + layers + mkPackage + ; +} diff --git a/pkgs/by-name/he/hello/package.nix b/pkgs/by-name/he/hello/package.nix index 0590131913f46..8f7b7b85a3ef4 100644 --- a/pkgs/by-name/he/hello/package.nix +++ b/pkgs/by-name/he/hello/package.nix @@ -1,59 +1,71 @@ -{ - callPackage, - lib, - stdenv, - fetchurl, - nixos, - testers, - versionCheckHook, - hello, -}: +{ mkPackage }: +mkPackage ( + { + stdenv, + fetchurl, + testers, + layers, + lib, + callPackage, + versionCheckHook, + }: + [ + (layers.derivation { inherit stdenv; }) + (this: old: { + name = "hello"; + version = "2.12.1"; -stdenv.mkDerivation (finalAttrs: { - pname = "hello"; - version = "2.12.1"; + setup = old.setup or { } // { + src = fetchurl { + url = "mirror://gnu/hello/hello-${this.version}.tar.gz"; + hash = "sha256-jZkUKv2SV28wsM18tCqNxoCZmLxdYH2Idh9RLibH2yA="; + }; - src = fetchurl { - url = "mirror://gnu/hello/hello-${finalAttrs.version}.tar.gz"; - hash = "sha256-jZkUKv2SV28wsM18tCqNxoCZmLxdYH2Idh9RLibH2yA="; - }; + doCheck = true; + doInstallCheck = true; + nativeInstallCheckInputs = [ + versionCheckHook + ]; - # The GNU Hello `configure` script detects how to link libiconv but fails to actually make use of that. - # Unfortunately, this cannot be a patch to `Makefile.am` because `autoreconfHook` causes a gettext - # infrastructure mismatch error when trying to build `hello`. - env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { - NIX_LDFLAGS = "-liconv"; - }; + # Give hello some install checks for testing purpose. + postInstallCheck = '' + stat "''${!outputBin}/bin/${this.meta.mainProgram}" + ''; + }; - doCheck = true; + # The GNU Hello `configure` script detects how to link libiconv but fails to actually make use of that. + # Unfortunately, this cannot be a patch to `Makefile.am` because `autoreconfHook` causes a gettext + # infrastructure mismatch error when trying to build `hello`. + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + NIX_LDFLAGS = "-liconv"; + }; - doInstallCheck = true; - nativeInstallCheckInputs = [ - versionCheckHook - ]; + public = old.public // { + tests = { + version = testers.testVersion { package = this.public; }; - # Give hello some install checks for testing purpose. - postInstallCheck = '' - stat "''${!outputBin}/bin/${finalAttrs.meta.mainProgram}" - ''; + run = callPackage ./test.nix { hello = this.public; }; + }; + # Other packages and tests expect to reuse our src + /** + Fetched sources of GNU hello + */ + inherit (this.setup) src; + }; - passthru.tests = { - version = testers.testVersion { package = hello; }; - }; - - passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.finalPackage; }; - - meta = { - description = "Program that produces a familiar, friendly greeting"; - longDescription = '' - GNU Hello is a program that prints "Hello, world!" when you run it. - It is fully customizable. - ''; - homepage = "https://www.gnu.org/software/hello/manual/"; - changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${finalAttrs.version}"; - license = lib.licenses.gpl3Plus; - maintainers = with lib.maintainers; [ stv0g ]; - mainProgram = "hello"; - platforms = lib.platforms.all; - }; -}) + meta = { + description = "Program that produces a familiar, friendly greeting"; + longDescription = '' + GNU Hello is a program that prints "Hello, world!" when you run it. + It is fully customizable. + ''; + homepage = "https://www.gnu.org/software/hello/manual/"; + changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${this.version}"; + license = lib.licenses.gpl3Plus; + maintainers = with lib.maintainers; [ stv0g ]; + mainProgram = "hello"; + platforms = lib.platforms.all; + }; + }) + ] +) diff --git a/pkgs/by-name/ne/netlify-cli/package.nix b/pkgs/by-name/ne/netlify-cli/package.nix index f766204b44770..ed92236688397 100644 --- a/pkgs/by-name/ne/netlify-cli/package.nix +++ b/pkgs/by-name/ne/netlify-cli/package.nix @@ -1,43 +1,58 @@ -{ - buildNpmPackage, - callPackage, - fetchFromGitHub, - lib, - nix-update-script, - nodejs, - pkg-config, - vips, -}: +{ mkPackage }: +mkPackage ( + { + lib, + layers, + stdenv, + fetchFromGitHub, + nix-update-script, + pkg-config, + vips, + callPackage, + testers, + nodejs_20, + ... + }: + [ + (layers.derivation { inherit stdenv; }) + (this: old: { + name = "netlify-cli"; + version = "18.0.0"; -buildNpmPackage rec { - pname = "netlify-cli"; - version = "18.0.0"; + deps = old.deps // { + nodejs = nodejs_20; + }; - src = fetchFromGitHub { - owner = "netlify"; - repo = "cli"; - tag = "v${version}"; - hash = "sha256-LGnFVg7c+CMgjxkVdy/rdoo6uU5HaOwGKRDHRe5Hz3Y="; - }; + npmFetch.hash = "sha256-ONLkCbmmY45/sRwaGUWhA187YVtCcdPVnD7ZMFoQ2Y0="; - npmDepsHash = "sha256-ONLkCbmmY45/sRwaGUWhA187YVtCcdPVnD7ZMFoQ2Y0="; + setup = { + buildInputs = [ vips ]; + nativeBuildInputs = [ pkg-config ]; + src = fetchFromGitHub { + owner = "netlify"; + repo = "cli"; + tag = "v${this.version}"; + hash = "sha256-LGnFVg7c+CMgjxkVdy/rdoo6uU5HaOwGKRDHRe5Hz3Y="; + }; + }; - inherit nodejs; + public = old.public // { + tests = { + version = testers.testVersion { package = this.public; }; + run = callPackage ./test.nix { netlify-cli = this.public; }; + }; + updateScript = nix-update-script { }; + }; - buildInputs = [ vips ]; - nativeBuildInputs = [ pkg-config ]; - - passthru = { - tests.test = callPackage ./test.nix { }; - updateScript = nix-update-script { }; - }; - - meta = { - description = "Netlify command line tool"; - homepage = "https://github.com/netlify/cli"; - changelog = "https://github.com/netlify/cli/blob/v${version}/CHANGELOG.md"; - license = lib.licenses.mit; - maintainers = with lib.maintainers; [ roberth ]; - mainProgram = "netlify"; - }; -} + meta = { + description = "Netlify command line tool"; + homepage = "https://github.com/netlify/cli"; + changelog = "https://github.com/netlify/cli/blob/v${this.version}/CHANGELOG.md"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ roberth ]; + mainProgram = "netlify"; + }; + }) + layers.buildNpmPackage + ] +) diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index fe9843c6b1202..ca78932ecbc63 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -63,6 +63,10 @@ let # This is convient to have as a parameter so the stdenv "adapters" work better mkDerivationFromStdenv ? stdenv: (import ./make-derivation.nix { inherit lib config; } stdenv).mkDerivation, + + # TODO: unify with the above + makeDerivationArgumentFromStdenv ? + stdenv: (import ./make-derivation.nix { inherit lib config; } stdenv).makeDerivationArgument, }: let @@ -201,6 +205,8 @@ let mkDerivation = mkDerivationFromStdenv stdenv; + makeDerivationArgument = makeDerivationArgumentFromStdenv stdenv; + inherit fetchurlBoot; inherit overrides; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index d88ba53fbb475..0de4d494f5862 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -677,4 +677,5 @@ extendDerivation in { inherit mkDerivation; + inherit makeDerivationArgument; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 2a9a047d7768e..0e9b878fbeb12 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -115,6 +115,11 @@ with pkgs; import ./pkg-config/defaultPkgConfigPackages.nix pkgs ) // { __attrsFailEvaluation = true; }; + inherit (import ../build-support/package/make-package.nix { inherit callPackage config lib stdenv; }) + layers + mkPackage + ; + ### Nixpkgs maintainer tools nix-generate-from-cpan = callPackage ../../maintainers/scripts/nix-generate-from-cpan.nix { }; @@ -4328,10 +4333,6 @@ with pkgs; }; }); - netlify-cli = callPackage ../by-name/ne/netlify-cli/package.nix { - nodejs = nodejs_20; - }; - netpbm = callPackage ../tools/graphics/netpbm { }; networkmanager = callPackage ../tools/networking/networkmanager { };