From 33282cd9f5ee740fd248a581767fc0d5ecc35845 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 9 Dec 2024 23:22:12 +0100 Subject: [PATCH 1/5] ci/eval: add rebuildsByPlatform to the comparison result --- ci/eval/compare/default.nix | 69 +++++++++++- ci/eval/compare/utils.nix | 203 +++++++++++++++++++++++++----------- 2 files changed, 207 insertions(+), 65 deletions(-) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 8b5e9059cd5b8..4f3b943a5a135 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -3,14 +3,66 @@ jq, runCommand, writeText, - supportedSystems, ... }: { beforeResultDir, afterResultDir }: let + /* + Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs. + Note: "platforms" are "x86_64-linux", "aarch64-darwin", ... + + --- + Inputs: + - beforeResultDir, afterResultDir: The evaluation result from before and after the change. + They can be obtained by running `nix-build -A ci.eval.full` on both revisions. + + --- + Outputs: + - changed-paths.json: Various information about the changes: + { + attrdiff: { + added: ["package1"], + changed: ["package2", "package3"], + removed: ["package4"], + }, + labels: [ + "10.rebuild-darwin: 1-10", + "10.rebuild-linux: 1-10" + ], + rebuildsByKernel: { + darwin: ["package1", "package2"], + linux: ["package1", "package2", "package3"] + }, + rebuildCountByKernel: { + darwin: 2, + linux: 3, + }, + rebuildsByPlatform: { + aarch64-darwin: ["package1", "package2"], + aarch64-linux: ["package1", "package2"], + x86_64-linux: ["package1", "package2", "package3"], + x86_64-darwin: ["package1"], + }, + } + - step-summary.md: A markdown render of the changes + + --- + Implementation details: + + Helper functions can be found in ./utils.nix. + Two main "types" are important: + + - `packagePlatformPath`: A string of the form "." + Example: "python312Packages.numpy.x86_64-linux" + + - `packagePlatformAttr`: An attrs representation of a packagePlatformPath: + Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; } + */ inherit (import ./utils.nix { inherit lib; }) diff groupByKernel + convertToPackagePlatformAttrs + groupByPlatform extractPackageNames getLabels uniqueStrings @@ -20,21 +72,30 @@ let beforeAttrs = getAttrs beforeResultDir; afterAttrs = getAttrs afterResultDir; + # Attrs + # - keys: "added", "changed" and "removed" + # - values: lists of `packagePlatformPath`s diffAttrs = diff beforeAttrs afterAttrs; changed-paths = let rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed); + rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds; - rebuildsByKernel = groupByKernel rebuilds; + rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs; + rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs; rebuildCountByKernel = lib.mapAttrs ( kernel: kernelRebuilds: lib.length kernelRebuilds ) rebuildsByKernel; in writeText "changed-paths.json" ( builtins.toJSON { - attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs; - inherit rebuildsByKernel rebuildCountByKernel; + attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs; + inherit + rebuildsByPlatform + rebuildsByKernel + rebuildCountByKernel + ; labels = getLabels rebuildCountByKernel; } ); diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix index 2794b5eda4ec3..d9b773b3fb324 100644 --- a/ci/eval/compare/utils.nix +++ b/ci/eval/compare/utils.nix @@ -3,55 +3,101 @@ rec { # Borrowed from https://github.com/NixOS/nixpkgs/pull/355616 uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list); - _processSystemPath = - packageSystemPath: + /* + Converts a `packagePlatformPath` into a `packagePlatformAttr` + + Turns + "hello.aarch64-linux" + into + { + name = "hello"; + platform = "aarch64-linux"; + } + */ + convertToPackagePlatformAttr = + packagePlatformPath: let - # python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"] - # splittedPath = lib.splitString "." attrName; - splittedPath = lib.splitString "." packageSystemPath; + # python312Packages.numpy.aarch64-linux -> ["python312Packages" "numpy" "aarch64-linux"] + splittedPath = lib.splitString "." packagePlatformPath; - # ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"] + # ["python312Packages" "numpy" "aarch64-linux"] -> ["python312Packages" "numpy"] packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath; - in - { - # "python312Packages.torch" + + # "python312Packages.numpy" name = lib.concatStringsSep "." packagePath; + in + if name == "" then + null + else + { + # python312Packages.numpy + inherit name; - # "aarch64-linux" - system = lib.last splittedPath; - }; + # "aarch64-linux" + platform = lib.last splittedPath; + }; - # Turns - # [ - # "hello.aarch64-linux" - # "hello.x86_64-linux" - # "hello.aarch64-darwin" - # "hello.x86_64-darwin" - # "bye.x86_64-darwin" - # "bye.aarch64-darwin" - # ] - # - # into - # - # [ - # "hello" - # "bye" - # ] + /* + Converts a list of `packagePlatformPath`s into a list of `packagePlatformAttr`s + + Turns + [ + "hello.aarch64-linux" + "hello.x86_64-linux" + "hello.aarch64-darwin" + "hello.x86_64-darwin" + "bye.x86_64-darwin" + "bye.aarch64-darwin" + "release-checks" <- Will be dropped + ] + into + [ + { name = "hello"; platform = "aarch64-linux"; } + { name = "hello"; platform = "x86_64-linux"; } + { name = "hello"; platform = "aarch64-darwin"; } + { name = "hello"; platform = "x86_64-darwin"; } + { name = "bye"; platform = "aarch64-darwin"; } + { name = "bye"; platform = "x86_64-darwin"; } + ] + */ + convertToPackagePlatformAttrs = + packagePlatformPaths: + builtins.filter (x: x != null) (builtins.map convertToPackagePlatformAttr packagePlatformPaths); + + /* + Converts a list of `packagePlatformPath`s directly to a list of (unique) package names + + Turns + [ + "hello.aarch64-linux" + "hello.x86_64-linux" + "hello.aarch64-darwin" + "hello.x86_64-darwin" + "bye.x86_64-darwin" + "bye.aarch64-darwin" + ] + into + [ + "hello" + "bye" + ] + */ extractPackageNames = - packageSystemPaths: - builtins.attrNames ( - builtins.removeAttrs (builtins.groupBy ( - packageSystemPath: (_processSystemPath packageSystemPath).name - ) packageSystemPaths) [ "" ] - ); - - # Computes a diff between two attrs - # { - # added: [ ], - # removed: [ ], - # changed: [ ], - # } - # + packagePlatformPaths: + let + packagePlatformAttrs = convertToPackagePlatformAttrs (uniqueStrings packagePlatformPaths); + in + builtins.map (p: p.name) packagePlatformAttrs; + + /* + Computes the key difference between two attrs + + { + added: [ ], + removed: [ ], + changed: [ ], + } + */ diff = let filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs); @@ -69,43 +115,78 @@ rec { ) old; }; + /* + Group a list of `packagePlatformAttr`s by platforms + + Turns + [ + { name = "hello"; platform = "aarch64-linux"; } + { name = "hello"; platform = "x86_64-linux"; } + { name = "hello"; platform = "aarch64-darwin"; } + { name = "hello"; platform = "x86_64-darwin"; } + { name = "bye"; platform = "aarch64-darwin"; } + { name = "bye"; platform = "x86_64-darwin"; } + ] + into + { + aarch64-linux = [ "hello" ]; + x86_64-linux = [ "hello" ]; + aarch64-darwin = [ "hello" "bye" ]; + x86_64-darwin = [ "hello" "bye" ]; + } + */ + groupByPlatform = + packagePlatformAttrs: + let + packagePlatformAttrsByPlatform = builtins.groupBy (p: p.platform) packagePlatformAttrs; + extractPackageNames = map (p: p.name); + in + lib.mapAttrs (_: extractPackageNames) packagePlatformAttrsByPlatform; + # Turns # [ - # "hello.aarch64-linux" - # "hello.x86_64-linux" - # "hello.aarch64-darwin" - # "hello.x86_64-darwin" - # "bye.x86_64-darwin" - # "bye.aarch64-darwin" + # { name = "hello"; platform = "aarch64-linux"; } + # { name = "hello"; platform = "x86_64-linux"; } + # { name = "hello"; platform = "aarch64-darwin"; } + # { name = "hello"; platform = "x86_64-darwin"; } + # { name = "bye"; platform = "aarch64-darwin"; } + # { name = "bye"; platform = "x86_64-darwin"; } # ] # # into # # { - # linux = [ - # "hello" - # ]; - # darwin = [ - # "hello" - # "bye" - # ]; + # linux = [ "hello" ]; + # darwin = [ "hello" "bye" ]; # } groupByKernel = - systemPaths: + packagePlatformAttrs: let - systemPaths' = builtins.map _processSystemPath systemPaths; - filterKernel = kernel: builtins.attrNames ( - builtins.groupBy (systemPath: systemPath.name) ( - builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths' + builtins.groupBy (p: p.name) ( + builtins.filter (p: lib.hasSuffix kernel p.platform) packagePlatformAttrs ) ); in lib.genAttrs [ "linux" "darwin" ] filterKernel; - getLabels = lib.mapAttrs ( + /* + Maps an attrs of `kernel - rebuild counts` mappings to a list of labels + + Turns + { + linux = 56; + darwin = 8; + } + into + [ + "10.rebuild-darwin: 1-10" + "10.rebuild-linux: 11-100" + ] + */ + getLabels = lib.mapAttrsToList ( kernel: rebuildCount: let number = From 4c7855b228eb8f7cbdc557b3acb80749f88df9d4 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 11 Dec 2024 16:05:43 +0100 Subject: [PATCH 2/5] materialize: remove darwin platforms --- pkgs/by-name/ma/materialize/package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/by-name/ma/materialize/package.nix b/pkgs/by-name/ma/materialize/package.nix index c3745ba58f475..7f5393b028908 100644 --- a/pkgs/by-name/ma/materialize/package.nix +++ b/pkgs/by-name/ma/materialize/package.nix @@ -190,6 +190,7 @@ rustPlatform.buildRustPackage rec { description = "Streaming SQL materialized view engine for real-time applications"; license = lib.licenses.bsl11; platforms = lib.platforms.unix; + badPlatform = lib.platforms.darwin; maintainers = with lib.maintainers; [ petrosagg ]; mainProgram = "environmentd"; }; From 307b8d135fc8010d1ead8840f86e46faa4130fb0 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 11 Dec 2024 16:07:30 +0100 Subject: [PATCH 3/5] python312Packages.equinox: remove --- pkgs/development/python-modules/equinox/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/development/python-modules/equinox/default.nix b/pkgs/development/python-modules/equinox/default.nix index 8b9b57a6c8b3f..eefd4ba8c0371 100644 --- a/pkgs/development/python-modules/equinox/default.nix +++ b/pkgs/development/python-modules/equinox/default.nix @@ -40,6 +40,10 @@ buildPythonPackage rec { typing-extensions ]; + postCheck = '' + echo OUI + ''; + nativeCheckInputs = [ beartype optax From edced816128844f4ece6c6ef9315ff334ab4d523 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 11 Dec 2024 16:08:27 +0100 Subject: [PATCH 4/5] qmake3demo: remove --- pkgs/top-level/all-packages.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f3c331d76a5f1..cb11c228e44b2 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -16984,12 +16984,6 @@ with pkgs; quake3wrapper = callPackage ../games/quake3/wrapper { }; - quake3demo = quake3wrapper { - name = "quake3-demo-${lib.getVersion quake3demodata}"; - description = "Demo of Quake 3 Arena, a classic first-person shooter"; - paks = [ quake3pointrelease quake3demodata ]; - }; - quake3demodata = callPackage ../games/quake3/content/demo.nix { }; quake3pointrelease = callPackage ../games/quake3/content/pointrelease.nix { }; From aea0424b31df8e83dde85c686c72e9282ac1ebab Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 11 Dec 2024 16:09:10 +0100 Subject: [PATCH 5/5] gh-cal-bis: init --- pkgs/top-level/all-packages.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index cb11c228e44b2..24810f5f73259 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1012,6 +1012,7 @@ with pkgs; }); gh-cal = callPackage ../tools/misc/gh-cal { }; + gh-cal-bis = callPackage ../tools/misc/gh-cal { }; gp-saml-gui = python3Packages.callPackage ../tools/networking/gp-saml-gui { };