Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Foo #364254

Closed
wants to merge 6 commits into from
Closed

Foo #364254

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 65 additions & 4 deletions ci/eval/compare/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,66 @@
jq,
runCommand,
writeText,
supportedSystems,
...
}:
{ beforeResultDir, afterResultDir }:
let
/*
Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...

---
Inputs:
- beforeResultDir, afterResultDir: The evaluation result from before and after the change.
They can be obtained by running `nix-build -A ci.eval.full` on both revisions.

---
Outputs:
- changed-paths.json: Various information about the changes:
{
attrdiff: {
added: ["package1"],
changed: ["package2", "package3"],
removed: ["package4"],
},
labels: [
"10.rebuild-darwin: 1-10",
"10.rebuild-linux: 1-10"
],
rebuildsByKernel: {
darwin: ["package1", "package2"],
linux: ["package1", "package2", "package3"]
},
rebuildCountByKernel: {
darwin: 2,
linux: 3,
},
rebuildsByPlatform: {
aarch64-darwin: ["package1", "package2"],
aarch64-linux: ["package1", "package2"],
x86_64-linux: ["package1", "package2", "package3"],
x86_64-darwin: ["package1"],
},
}
- step-summary.md: A markdown render of the changes

---
Implementation details:

Helper functions can be found in ./utils.nix.
Two main "types" are important:

- `packagePlatformPath`: A string of the form "<PACKAGE_PATH>.<PLATFORM>"
Example: "python312Packages.numpy.x86_64-linux"

- `packagePlatformAttr`: An attrs representation of a packagePlatformPath:
Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; }
*/
inherit (import ./utils.nix { inherit lib; })
diff
groupByKernel
convertToPackagePlatformAttrs
groupByPlatform
extractPackageNames
getLabels
uniqueStrings
Expand All @@ -20,21 +72,30 @@ let
beforeAttrs = getAttrs beforeResultDir;
afterAttrs = getAttrs afterResultDir;

# Attrs
# - keys: "added", "changed" and "removed"
# - values: lists of `packagePlatformPath`s
diffAttrs = diff beforeAttrs afterAttrs;

changed-paths =
let
rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;

rebuildsByKernel = groupByKernel rebuilds;
rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs;
rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs;
rebuildCountByKernel = lib.mapAttrs (
kernel: kernelRebuilds: lib.length kernelRebuilds
) rebuildsByKernel;
in
writeText "changed-paths.json" (
builtins.toJSON {
attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs;
inherit rebuildsByKernel rebuildCountByKernel;
attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs;
inherit
rebuildsByPlatform
rebuildsByKernel
rebuildCountByKernel
;
labels = getLabels rebuildCountByKernel;
}
);
Expand Down
203 changes: 142 additions & 61 deletions ci/eval/compare/utils.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,55 +3,101 @@ rec {
# Borrowed from https://github.com/NixOS/nixpkgs/pull/355616
uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);

_processSystemPath =
packageSystemPath:
/*
Converts a `packagePlatformPath` into a `packagePlatformAttr`

Turns
"hello.aarch64-linux"
into
{
name = "hello";
platform = "aarch64-linux";
}
*/
convertToPackagePlatformAttr =
packagePlatformPath:
let
# python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"]
# splittedPath = lib.splitString "." attrName;
splittedPath = lib.splitString "." packageSystemPath;
# python312Packages.numpy.aarch64-linux -> ["python312Packages" "numpy" "aarch64-linux"]
splittedPath = lib.splitString "." packagePlatformPath;

# ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"]
# ["python312Packages" "numpy" "aarch64-linux"] -> ["python312Packages" "numpy"]
packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath;
in
{
# "python312Packages.torch"

# "python312Packages.numpy"
name = lib.concatStringsSep "." packagePath;
in
if name == "" then
null
else
{
# python312Packages.numpy
inherit name;

# "aarch64-linux"
system = lib.last splittedPath;
};
# "aarch64-linux"
platform = lib.last splittedPath;
};

# Turns
# [
# "hello.aarch64-linux"
# "hello.x86_64-linux"
# "hello.aarch64-darwin"
# "hello.x86_64-darwin"
# "bye.x86_64-darwin"
# "bye.aarch64-darwin"
# ]
#
# into
#
# [
# "hello"
# "bye"
# ]
/*
Converts a list of `packagePlatformPath`s into a list of `packagePlatformAttr`s

Turns
[
"hello.aarch64-linux"
"hello.x86_64-linux"
"hello.aarch64-darwin"
"hello.x86_64-darwin"
"bye.x86_64-darwin"
"bye.aarch64-darwin"
"release-checks" <- Will be dropped
]
into
[
{ name = "hello"; platform = "aarch64-linux"; }
{ name = "hello"; platform = "x86_64-linux"; }
{ name = "hello"; platform = "aarch64-darwin"; }
{ name = "hello"; platform = "x86_64-darwin"; }
{ name = "bye"; platform = "aarch64-darwin"; }
{ name = "bye"; platform = "x86_64-darwin"; }
]
*/
convertToPackagePlatformAttrs =
packagePlatformPaths:
builtins.filter (x: x != null) (builtins.map convertToPackagePlatformAttr packagePlatformPaths);

/*
Converts a list of `packagePlatformPath`s directly to a list of (unique) package names

Turns
[
"hello.aarch64-linux"
"hello.x86_64-linux"
"hello.aarch64-darwin"
"hello.x86_64-darwin"
"bye.x86_64-darwin"
"bye.aarch64-darwin"
]
into
[
"hello"
"bye"
]
*/
extractPackageNames =
packageSystemPaths:
builtins.attrNames (
builtins.removeAttrs (builtins.groupBy (
packageSystemPath: (_processSystemPath packageSystemPath).name
) packageSystemPaths) [ "" ]
);

# Computes a diff between two attrs
# {
# added: [ <keys only in the second object> ],
# removed: [ <keys only in the first object> ],
# changed: [ <keys with different values between the two objects> ],
# }
#
packagePlatformPaths:
let
packagePlatformAttrs = convertToPackagePlatformAttrs (uniqueStrings packagePlatformPaths);
in
builtins.map (p: p.name) packagePlatformAttrs;

/*
Computes the key difference between two attrs

{
added: [ <keys only in the second object> ],
removed: [ <keys only in the first object> ],
changed: [ <keys with different values between the two objects> ],
}
*/
diff =
let
filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
Expand All @@ -69,43 +115,78 @@ rec {
) old;
};

/*
Group a list of `packagePlatformAttr`s by platforms

Turns
[
{ name = "hello"; platform = "aarch64-linux"; }
{ name = "hello"; platform = "x86_64-linux"; }
{ name = "hello"; platform = "aarch64-darwin"; }
{ name = "hello"; platform = "x86_64-darwin"; }
{ name = "bye"; platform = "aarch64-darwin"; }
{ name = "bye"; platform = "x86_64-darwin"; }
]
into
{
aarch64-linux = [ "hello" ];
x86_64-linux = [ "hello" ];
aarch64-darwin = [ "hello" "bye" ];
x86_64-darwin = [ "hello" "bye" ];
}
*/
groupByPlatform =
packagePlatformAttrs:
let
packagePlatformAttrsByPlatform = builtins.groupBy (p: p.platform) packagePlatformAttrs;
extractPackageNames = map (p: p.name);
in
lib.mapAttrs (_: extractPackageNames) packagePlatformAttrsByPlatform;

# Turns
# [
# "hello.aarch64-linux"
# "hello.x86_64-linux"
# "hello.aarch64-darwin"
# "hello.x86_64-darwin"
# "bye.x86_64-darwin"
# "bye.aarch64-darwin"
# { name = "hello"; platform = "aarch64-linux"; }
# { name = "hello"; platform = "x86_64-linux"; }
# { name = "hello"; platform = "aarch64-darwin"; }
# { name = "hello"; platform = "x86_64-darwin"; }
# { name = "bye"; platform = "aarch64-darwin"; }
# { name = "bye"; platform = "x86_64-darwin"; }
# ]
#
# into
#
# {
# linux = [
# "hello"
# ];
# darwin = [
# "hello"
# "bye"
# ];
# linux = [ "hello" ];
# darwin = [ "hello" "bye" ];
# }
groupByKernel =
systemPaths:
packagePlatformAttrs:
let
systemPaths' = builtins.map _processSystemPath systemPaths;

filterKernel =
kernel:
builtins.attrNames (
builtins.groupBy (systemPath: systemPath.name) (
builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths'
builtins.groupBy (p: p.name) (
builtins.filter (p: lib.hasSuffix kernel p.platform) packagePlatformAttrs
)
);
in
lib.genAttrs [ "linux" "darwin" ] filterKernel;

getLabels = lib.mapAttrs (
/*
Maps an attrs of `kernel - rebuild counts` mappings to a list of labels

Turns
{
linux = 56;
darwin = 8;
}
into
[
"10.rebuild-darwin: 1-10"
"10.rebuild-linux: 11-100"
]
*/
getLabels = lib.mapAttrsToList (
kernel: rebuildCount:
let
number =
Expand Down
1 change: 1 addition & 0 deletions pkgs/by-name/ma/materialize/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ rustPlatform.buildRustPackage rec {
description = "Streaming SQL materialized view engine for real-time applications";
license = lib.licenses.bsl11;
platforms = lib.platforms.unix;
badPlatform = lib.platforms.darwin;
maintainers = with lib.maintainers; [ petrosagg ];
mainProgram = "environmentd";
};
Expand Down
4 changes: 4 additions & 0 deletions pkgs/development/python-modules/equinox/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ buildPythonPackage rec {
typing-extensions
];

postCheck = ''
echo OUI
'';

nativeCheckInputs = [
beartype
optax
Expand Down
7 changes: 1 addition & 6 deletions pkgs/top-level/all-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,7 @@ with pkgs;
});

gh-cal = callPackage ../tools/misc/gh-cal { };
gh-cal-bis = callPackage ../tools/misc/gh-cal { };

gp-saml-gui = python3Packages.callPackage ../tools/networking/gp-saml-gui { };

Expand Down Expand Up @@ -16984,12 +16985,6 @@ with pkgs;

quake3wrapper = callPackage ../games/quake3/wrapper { };

quake3demo = quake3wrapper {
name = "quake3-demo-${lib.getVersion quake3demodata}";
description = "Demo of Quake 3 Arena, a classic first-person shooter";
paks = [ quake3pointrelease quake3demodata ];
};

quake3demodata = callPackage ../games/quake3/content/demo.nix { };

quake3pointrelease = callPackage ../games/quake3/content/pointrelease.nix { };
Expand Down
Loading