From 6468b7adc451183c97f2fcde5baafbfa269515de Mon Sep 17 00:00:00 2001 From: mib Date: Sun, 11 Aug 2024 19:06:54 +0200 Subject: [PATCH 01/10] lib.meta.licensesSpdx: mapping from SPDX ID to licenses --- lib/meta.nix | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/meta.nix b/lib/meta.nix index 65d7bf991..ddd062f38 100644 --- a/lib/meta.nix +++ b/lib/meta.nix @@ -286,6 +286,32 @@ rec { ((!pkg?meta.platforms) || any (platformMatch platform) pkg.meta.platforms) && all (elem: !platformMatch platform elem) (pkg.meta.badPlatforms or []); + /** + Mapping of SPDX ID to the attributes in lib.licenses. + + For SPDX IDs, see https://spdx.org/licenses + + # Examples + :::{.example} + ## `lib.meta.licensesSpdx` usage example + + ```nix + lib.licensesSpdx.MIT == lib.licenses.mit + => true + lib.licensesSpdx."MY LICENSE" + => error: attribute 'MY LICENSE' missing + ``` + + ::: + */ + licensesSpdx = + lib.attrsets.mapAttrs' + (_key: license: { + name = license.spdxId; + value = license; + }) + (lib.attrsets.filterAttrs (_key: license: license ? spdxId) lib.licenses); + /** Get the corresponding attribute in lib.licenses from the SPDX ID or warn and fallback to `{ shortName = ; }`. @@ -361,10 +387,12 @@ rec { */ getLicenseFromSpdxIdOr = let - spdxLicenses = lib.mapAttrs (id: ls: assert lib.length ls == 1; builtins.head ls) - (lib.groupBy (l: lib.toLower l.spdxId) (lib.filter (l: l ? spdxId) (lib.attrValues lib.licenses))); + lowercaseLicenses = lib.mapAttrs' (name: value: { + name = lib.toLower name; + inherit value; + }) licensesSpdx; in licstr: default: - spdxLicenses.${ lib.toLower licstr } or default; + lowercaseLicenses.${ lib.toLower licstr } or default; /** Get the path to the main program of a package based on meta.mainProgram From cf87e59781cb89e6dae8adad0de2f68a02fb6f05 Mon Sep 17 00:00:00 2001 From: Gabriella Gonzalez Date: Fri, 16 Aug 2024 16:26:08 +0200 Subject: [PATCH 02/10] lib.cli.escapeShellArg{,s}: Only escape when necessary (#333744) These utilities will now leave the string undisturbed if it doesn't need to be quoted (because it doesn't have any special characters). This can help generate nicer-looking command lines. This also transitively improves the output of `lib.toGNUCommandLine` which uses `escapeShellArg` internally --- lib/strings.nix | 16 ++++++++++++---- lib/tests/misc.nix | 28 ++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/strings.nix b/lib/strings.nix index aafbdffaa..54a5b9c67 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -1026,7 +1026,8 @@ rec { replaceStrings (builtins.attrNames toEscape) (lib.mapAttrsToList (_: c: "%${fixedWidthString 2 "0" (lib.toHexString c)}") toEscape); /** - Quote `string` to be used safely within the Bourne shell. + Quote `string` to be used safely within the Bourne shell if it has any + special characters. # Inputs @@ -1051,10 +1052,17 @@ rec { ::: */ - escapeShellArg = arg: "'${replaceStrings ["'"] ["'\\''"] (toString arg)}'"; + escapeShellArg = arg: + let + string = toString arg; + in + if match "[[:alnum:],._+:@%/-]+" string == null + then "'${replaceStrings ["'"] ["'\\''"] string}'" + else string; /** - Quote all arguments to be safely passed to the Bourne shell. + Quote all arguments that have special characters to be safely passed to the + Bourne shell. # Inputs @@ -1073,7 +1081,7 @@ rec { ```nix escapeShellArgs ["one" "two three" "four'five"] - => "'one' 'two three' 'four'\\''five'" + => "one 'two three' 'four'\\''five'" ``` ::: diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index d59f5586b..7445c6345 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -470,6 +470,26 @@ runTests { expected = [ "A" "B" ]; }; + testEscapeShellArg = { + expr = strings.escapeShellArg "esc'ape\nme"; + expected = "'esc'\\''ape\nme'"; + }; + + testEscapeShellArgEmpty = { + expr = strings.escapeShellArg ""; + expected = "''"; + }; + + testEscapeShellArgs = { + expr = strings.escapeShellArgs ["one" "two three" "four'five"]; + expected = "one 'two three' 'four'\\''five'"; + }; + + testEscapeShellArgsUnicode = { + expr = strings.escapeShellArg "á"; + expected = "'á'"; + }; + testSplitStringsDerivation = { expr = take 3 (strings.splitString "/" (derivation { name = "name"; @@ -569,12 +589,12 @@ runTests { ''; expected = '' STRing01='just a '\'''string'\'''' - declare -a _array_=('with' 'more strings') + declare -a _array_=(with 'more strings') declare -A assoc=(['with some']='strings possibly newlines ') - drv='/drv' - path='/path' + drv=/drv + path=/path stringable='hello toString' ''; }; @@ -1754,7 +1774,7 @@ runTests { verbose = true; }; - expected = "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"; + expected = "-X PUT --data '{\"id\":0}' --retry 3 --url https://example.com/foo --url https://example.com/bar --verbose"; }; testSanitizeDerivationNameLeadingDots = testSanitizeDerivationName { From b9a60272cbc4266c1e3a5c186a884e7c3f772370 Mon Sep 17 00:00:00 2001 From: Alexander Ben Nasrallah Date: Mon, 19 Aug 2024 11:28:50 +0200 Subject: [PATCH 03/10] lib.licenses: add CockroachDB Community License Agreement --- lib/licenses.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/licenses.nix b/lib/licenses.nix index 67744b499..2e1785dd9 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -403,6 +403,12 @@ lib.mapAttrs mkLicense ({ fullName = "CeCILL-C Free Software License Agreement"; }; + cockroachdb-community-license = { + fullName = "CockroachDB Community License Agreement"; + url = "https://www.cockroachlabs.com/cockroachdb-community-license/"; + free = false; + }; + cpal10 = { spdxId = "CPAL-1.0"; fullName = "Common Public Attribution License 1.0"; From ee1824485929e9ec1e0ee4e66707b091661355f3 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sun, 25 Aug 2024 00:57:44 +0200 Subject: [PATCH 04/10] lib.meta: Minor SPDX license function improvements - Expose `lib.licensesSpdx` - Create bindings for the needed internal functions - Mention that some SPDX licenses might be missing (in the future I hope we can autogenerate the Nixpkgs license list from some SPDX endpoint --- lib/default.nix | 3 ++- lib/meta.nix | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index 63a31101e..0ff3a3980 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -123,7 +123,8 @@ let inherit (self.derivations) lazyDerivation optionalDrvAttr; inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio - hiPrioSet getLicenseFromSpdxId getLicenseFromSpdxIdOr getExe getExe'; + hiPrioSet licensesSpdx getLicenseFromSpdxId getLicenseFromSpdxIdOr + getExe getExe'; inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile packagesFromDirectoryRecursive; inherit (self.sources) cleanSourceFilter diff --git a/lib/meta.nix b/lib/meta.nix index ddd062f38..57f3a46d0 100644 --- a/lib/meta.nix +++ b/lib/meta.nix @@ -7,6 +7,7 @@ let inherit (lib) matchAttrs any all isDerivation getBin assertMsg; + inherit (lib.attrsets) mapAttrs' filterAttrs; inherit (builtins) isString match typeOf; in @@ -289,7 +290,8 @@ rec { /** Mapping of SPDX ID to the attributes in lib.licenses. - For SPDX IDs, see https://spdx.org/licenses + For SPDX IDs, see https://spdx.org/licenses. + Note that some SPDX licenses might be missing. # Examples :::{.example} @@ -305,18 +307,19 @@ rec { ::: */ licensesSpdx = - lib.attrsets.mapAttrs' + mapAttrs' (_key: license: { name = license.spdxId; value = license; }) - (lib.attrsets.filterAttrs (_key: license: license ? spdxId) lib.licenses); + (filterAttrs (_key: license: license ? spdxId) lib.licenses); /** Get the corresponding attribute in lib.licenses from the SPDX ID or warn and fallback to `{ shortName = ; }`. - For SPDX IDs, see https://spdx.org/licenses + For SPDX IDs, see https://spdx.org/licenses. + Note that some SPDX licenses might be missing. # Type @@ -351,7 +354,8 @@ rec { Get the corresponding attribute in lib.licenses from the SPDX ID or fallback to the given default value. - For SPDX IDs, see https://spdx.org/licenses + For SPDX IDs, see https://spdx.org/licenses. + Note that some SPDX licenses might be missing. # Inputs From 7a07b7de17759c35ff2e761bd74f17b18bd99e54 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 12 Jul 2024 03:20:47 +0200 Subject: [PATCH 05/10] lib/tests: Format a file with nixfmt And fix locations to not break the test. This is a rare case where another change is required after formatting. We do this in a separate commit so that we don't need to do it in the treewide reformatting PR. --- lib/tests/modules.sh | 18 ++++++------- lib/tests/modules/declaration-positions.nix | 30 +++++++++++++-------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 3a23766a1..3301a3d98 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -542,21 +542,21 @@ checkConfigOutput '^"pear\\npear"$' config.twice.raw ./merge-module-with-key.nix # Declaration positions # Line should be present for direct options -checkConfigOutput '^10$' options.imported.line10.declarationPositions.0.line ./declaration-positions.nix -checkConfigOutput '/declaration-positions.nix"$' options.imported.line10.declarationPositions.0.file ./declaration-positions.nix +checkConfigOutput '^14$' options.imported.line14.declarationPositions.0.line ./declaration-positions.nix +checkConfigOutput '/declaration-positions.nix"$' options.imported.line14.declarationPositions.0.file ./declaration-positions.nix # Generated options may not have line numbers but they will at least get the # right file -checkConfigOutput '/declaration-positions.nix"$' options.generated.line18.declarationPositions.0.file ./declaration-positions.nix -checkConfigOutput '^null$' options.generated.line18.declarationPositions.0.line ./declaration-positions.nix +checkConfigOutput '/declaration-positions.nix"$' options.generated.line22.declarationPositions.0.file ./declaration-positions.nix +checkConfigOutput '^null$' options.generated.line22.declarationPositions.0.line ./declaration-positions.nix # Submodules don't break it -checkConfigOutput '^39$' config.submoduleLine34.submodDeclLine39.0.line ./declaration-positions.nix -checkConfigOutput '/declaration-positions.nix"$' config.submoduleLine34.submodDeclLine39.0.file ./declaration-positions.nix +checkConfigOutput '^45$' config.submoduleLine38.submodDeclLine45.0.line ./declaration-positions.nix +checkConfigOutput '/declaration-positions.nix"$' config.submoduleLine38.submodDeclLine45.0.file ./declaration-positions.nix # New options under freeform submodules get collected into the parent submodule # (consistent with .declarations behaviour, but weird; notably appears in system.build) -checkConfigOutput '^34|23$' options.submoduleLine34.declarationPositions.0.line ./declaration-positions.nix -checkConfigOutput '^34|23$' options.submoduleLine34.declarationPositions.1.line ./declaration-positions.nix +checkConfigOutput '^38|27$' options.submoduleLine38.declarationPositions.0.line ./declaration-positions.nix +checkConfigOutput '^38|27$' options.submoduleLine38.declarationPositions.1.line ./declaration-positions.nix # nested options work -checkConfigOutput '^30$' options.nested.nestedLine30.declarationPositions.0.line ./declaration-positions.nix +checkConfigOutput '^34$' options.nested.nestedLine34.declarationPositions.0.line ./declaration-positions.nix cat < Date: Wed, 17 Jul 2024 04:23:12 +0300 Subject: [PATCH 06/10] lib/systems: set gcc.arch = "armv6kz", gcc.fpu = "vfpv2" for raspberryPi >The CPU in the Raspberry Pi 1 and Zero implements the ARMv6 ISA (with VFP2) https://wiki.debian.org/RaspberryPi#:~:text=The%20CPU%20in%20the%20Raspberry%20Pi,VFP2%29 See also - https://en.wikipedia.org/wiki/Raspberry_Pi#Specifications - https://github.com/NixOS/nixpkgs/issues/319036#issuecomment-2232108373 --- lib/systems/platforms.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/systems/platforms.nix b/lib/systems/platforms.nix index 403ffc028..873c3c7fc 100644 --- a/lib/systems/platforms.nix +++ b/lib/systems/platforms.nix @@ -201,8 +201,9 @@ rec { target = "zImage"; }; gcc = { - arch = "armv6"; - fpu = "vfp"; + # https://en.wikipedia.org/wiki/Raspberry_Pi#Specifications + arch = "armv6kz"; + fpu = "vfpv2"; }; }; From 9cb2722c03a3b2460999cf399833df5ceca112bd Mon Sep 17 00:00:00 2001 From: Philip Taron Date: Thu, 29 Aug 2024 09:04:57 -0700 Subject: [PATCH 07/10] treewide: use a consistent meta.priority default --- lib/meta.nix | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/meta.nix b/lib/meta.nix index 57f3a46d0..798e7e749 100644 --- a/lib/meta.nix +++ b/lib/meta.nix @@ -133,12 +133,17 @@ rec { mapDerivationAttrset = f: set: lib.mapAttrs (name: pkg: if lib.isDerivation pkg then (f pkg) else pkg) set; /** - Set the nix-env priority of the package. + The default priority of packages in Nix. See `defaultPriority` in [`src/nix/profile.cc`](https://github.com/NixOS/nix/blob/master/src/nix/profile.cc#L47). + */ + defaultPriority = 5; + + /** + Set the nix-env priority of the package. Note that higher values are lower priority, and vice versa. # Inputs `priority` - : 1\. Function argument + : 1\. The priority to set. `drv` : 2\. Function argument @@ -159,8 +164,7 @@ rec { lowPrio = setPrio 10; /** - Apply lowPrio to an attrset with derivations - + Apply lowPrio to an attrset with derivations. # Inputs @@ -184,8 +188,7 @@ rec { hiPrio = setPrio (-10); /** - Apply hiPrio to an attrset with derivations - + Apply hiPrio to an attrset with derivations. # Inputs From 315acb494251bbdf61302ea58b5591b73359a3a6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 31 Aug 2024 01:12:43 +0200 Subject: [PATCH 08/10] lib.importApply: init (#230588) * lib.modules.importApply: init Brings variables from rich scopes to modules defined in separate files. A helper for functions in files that return a module. * lib.modules.importApply: Edit doc Generally improve the quality. Notes: - Not rendered to the manual yet, so probably the syntax could be improved, but I have no way to test this now. - The docs use `arg` vs `staticArg` in the code. This is intentional, because the doc is pretty clear about the role of `arg` whereas the code exists in a context where ambiguities are more harmful. * Format --- lib/modules.nix | 53 +++++++++++++++++++++ lib/tests/modules.sh | 8 ++++ lib/tests/modules/importApply-disabling.nix | 4 ++ lib/tests/modules/importApply-function.nix | 5 ++ lib/tests/modules/importApply.nix | 5 ++ 5 files changed, 75 insertions(+) create mode 100644 lib/tests/modules/importApply-disabling.nix create mode 100644 lib/tests/modules/importApply-function.nix create mode 100644 lib/tests/modules/importApply.nix diff --git a/lib/modules.nix b/lib/modules.nix index b9e9ca1e5..ef0be5b27 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -1366,6 +1366,58 @@ let ]); }; + /** + `importApply file arg :: Path -> a -> Module`, where `import file :: a -> Module` + + `importApply` imports a Nix expression file much like the module system would, + after passing an extra positional argument to the function in the file. + + This function should be used when declaring a module in a file that refers to + values from a different scope, such as that in a flake. + + It solves the problems of alternative solutions: + + - While `importApply file arg` is _mostly_ equivalent to + `import file arg`, the latter returns a module without a location, + as `import` only returns the contained expression. This leads to worse + error messages. + + - Using `specialArgs` to provide arguments to all modules. This effectively + creates an incomplete module, and requires the user of the module to + manually pass the `specialArgs` to the configuration, which is error-prone, + verbose, and unnecessary. + + The nix file must contain a function that returns a module. + A module may itself be a function, so the file is often a function with two + positional arguments instead of one. See the example below. + + This function does not add support for deduplication and `disabledModules`, + although that could be achieved by wrapping the returned module and setting + the `_key` module attribute. + The reason for this omission is that the file path is not guaranteed to be + a unique identifier for the module, as two instances of the module may + reference different `arg`s in their closures. + + Example + + # lib.nix + imports = [ + (lib.modules.importApply ./module.nix { bar = bar; }) + ]; + + # module.nix + { bar }: + { lib, config, ... }: + { + options = ...; + config = ... bar ...; + } + + */ + importApply = + modulePath: staticArg: + lib.setDefaultModuleLocation modulePath (import modulePath staticArg); + /* Use this function to import a JSON file as NixOS configuration. modules.importJSON :: path -> attrs @@ -1415,6 +1467,7 @@ private // filterOverrides' fixMergeModules fixupOptionType # should be private? + importApply importJSON importTOML mergeDefinitions diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 3301a3d98..09547b2e0 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -247,6 +247,14 @@ checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-if-foo-e checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-if-enable.nix checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-enable-if.nix +# Check importApply +checkConfigOutput '"abc"' config.value ./importApply.nix +# importApply does not set a key. +# Disabling the function file is not sufficient, because importApply can't reasonably assume that the key is unique. +# e.g. user may call it multiple times with different arguments and expect each of the module to apply. +# While this is excusable for the disabledModules aspect, it is not for the deduplication of modules. +checkConfigOutput '"abc"' config.value ./importApply-disabling.nix + # Check disabledModules with config definitions and option declarations. set -- config.enable ./define-enable.nix ./declare-enable.nix checkConfigOutput '^true$' "$@" diff --git a/lib/tests/modules/importApply-disabling.nix b/lib/tests/modules/importApply-disabling.nix new file mode 100644 index 000000000..016036dd8 --- /dev/null +++ b/lib/tests/modules/importApply-disabling.nix @@ -0,0 +1,4 @@ +{ + imports = [ ./importApply.nix ]; + disabledModules = [ ./importApply-function.nix ]; +} diff --git a/lib/tests/modules/importApply-function.nix b/lib/tests/modules/importApply-function.nix new file mode 100644 index 000000000..7c193a912 --- /dev/null +++ b/lib/tests/modules/importApply-function.nix @@ -0,0 +1,5 @@ +{ foo }: +{ lib, config, ... }: +{ + value = foo; +} diff --git a/lib/tests/modules/importApply.nix b/lib/tests/modules/importApply.nix new file mode 100644 index 000000000..d3b893cb3 --- /dev/null +++ b/lib/tests/modules/importApply.nix @@ -0,0 +1,5 @@ +{ lib, ... }: +{ + options.value = lib.mkOption { default = 1; }; + imports = [ (lib.modules.importApply ./importApply-function.nix { foo = "abc"; }) ]; +} From 552137d9b857c09cbf34403326fe6dafdcc0900e Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Fri, 30 Aug 2024 08:34:22 +0200 Subject: [PATCH 09/10] lib.platforms.mesaPlatforms: remove Mesa is a package like any other. There's no reason for it to be a special case with its platforms listed in lib, because if other packages want to refer to mesa's platforms, they can access the platforms from the package meta like they would for any other package. --- lib/systems/doubles.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/systems/doubles.nix b/lib/systems/doubles.nix index b4cd5acfc..bf4706bbf 100644 --- a/lib/systems/doubles.nix +++ b/lib/systems/doubles.nix @@ -114,6 +114,4 @@ in { genode = filterDoubles predicates.isGenode; embedded = filterDoubles predicates.isNone; - - mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "armv7a-linux" "aarch64-linux" "powerpc64-linux" "powerpc64le-linux" "aarch64-darwin" "riscv64-linux"]; } From bfdd658e678369986219c9c309972aff5ea7e385 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 15 Jul 2024 14:38:31 +0200 Subject: [PATCH 10/10] lib.platforms.aarch: init --- lib/systems/doubles.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/systems/doubles.nix b/lib/systems/doubles.nix index bf4706bbf..6939e9a40 100644 --- a/lib/systems/doubles.nix +++ b/lib/systems/doubles.nix @@ -69,6 +69,7 @@ in { arm = filterDoubles predicates.isAarch32; armv7 = filterDoubles predicates.isArmv7; + aarch = filterDoubles predicates.isAarch; aarch64 = filterDoubles predicates.isAarch64; x86 = filterDoubles predicates.isx86; i686 = filterDoubles predicates.isi686;