gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci/eval: add rebuildsByPlatform to the comparison result

Browse Source
This commit is contained in:
Gaetan Lepage
2024-12-09 23:22:12 +01:00
parent b3438737b4
commit 518ae8fd58
2 changed files with 206 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
ci/eval/compare/default.nix
View File

@@ -3,14 +3,66 @@
jq,
runCommand,
writeText,
supportedSystems,
...
}:
{ beforeResultDir, afterResultDir }:
let
/*
Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...
---
Inputs:
- beforeResultDir, afterResultDir: The evaluation result from before and after the change.
They can be obtained by running `nix-build -A ci.eval.full` on both revisions.
---
Outputs:
- changed-paths.json: Various information about the changes:
{
attrdiff: {
added: ["package1"],
changed: ["package2", "package3"],
removed: ["package4"],
},
labels: [
"10.rebuild-darwin: 1-10",
"10.rebuild-linux: 1-10"
],
rebuildsByKernel: {
darwin: ["package1", "package2"],
linux: ["package1", "package2", "package3"]
},
rebuildCountByKernel: {
darwin: 2,
linux: 3,
},
rebuildsByPlatform: {
aarch64-darwin: ["package1", "package2"],
aarch64-linux: ["package1", "package2"],
x86_64-linux: ["package1", "package2", "package3"],
x86_64-darwin: ["package1"],
},
}
- step-summary.md: A markdown render of the changes
---
Implementation details:
Helper functions can be found in ./utils.nix.
Two main "types" are important:
- `packagePlatformPath`: A string of the form "<PACKAGE_PATH>.<PLATFORM>"
Example: "python312Packages.numpy.x86_64-linux"
- `packagePlatformAttr`: An attrs representation of a packagePlatformPath:
Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; }
*/
inherit (import ./utils.nix { inherit lib; })
diff
groupByKernel
convertToPackagePlatformAttrs
groupByPlatform
extractPackageNames
getLabels
uniqueStrings
@@ -20,21 +72,30 @@ let
beforeAttrs = getAttrs beforeResultDir;
afterAttrs = getAttrs afterResultDir;
# Attrs
# - keys: "added", "changed" and "removed"
# - values: lists of `packagePlatformPath`s
diffAttrs = diff beforeAttrs afterAttrs;
changed-paths =
let
rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
rebuildsByKernel = groupByKernel rebuilds;
rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs;
rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs;
rebuildCountByKernel = lib.mapAttrs (
kernel: kernelRebuilds: lib.length kernelRebuilds
) rebuildsByKernel;
in
writeText "changed-paths.json" (
builtins.toJSON {
attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs;
inherit rebuildsByKernel rebuildCountByKernel;
attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs;
inherit
rebuildsByPlatform
rebuildsByKernel
rebuildCountByKernel
;
labels = getLabels rebuildCountByKernel;
}
);

201
ci/eval/compare/utils.nix
View File

@@ -3,55 +3,101 @@ rec {
# Borrowed from https://github.com/NixOS/nixpkgs/pull/355616
uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
_processSystemPath =
packageSystemPath:
/*
Converts a `packagePlatformPath` into a `packagePlatformAttr`
Turns
"hello.aarch64-linux"
into
{
name = "hello";
platform = "aarch64-linux";
}
*/
convertToPackagePlatformAttr =
packagePlatformPath:
let
# python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"]
# splittedPath = lib.splitString "." attrName;
splittedPath = lib.splitString "." packageSystemPath;
# python312Packages.numpy.aarch64-linux -> ["python312Packages" "numpy" "aarch64-linux"]
splittedPath = lib.splitString "." packagePlatformPath;
# ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"]
# ["python312Packages" "numpy" "aarch64-linux"] -> ["python312Packages" "numpy"]
packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath;
in
{
# "python312Packages.torch"
# "python312Packages.numpy"
name = lib.concatStringsSep "." packagePath;
in
if name == "" then
null
else
{
# python312Packages.numpy
inherit name;
# "aarch64-linux"
system = lib.last splittedPath;
};
# "aarch64-linux"
platform = lib.last splittedPath;
};
# Turns
# [
# "hello.aarch64-linux"
# "hello.x86_64-linux"
# "hello.aarch64-darwin"
# "hello.x86_64-darwin"
# "bye.x86_64-darwin"
# "bye.aarch64-darwin"
# ]
#
# into
#
# [
# "hello"
# "bye"
# ]
/*
Converts a list of `packagePlatformPath`s into a list of `packagePlatformAttr`s
Turns
[
"hello.aarch64-linux"
"hello.x86_64-linux"
"hello.aarch64-darwin"
"hello.x86_64-darwin"
"bye.x86_64-darwin"
"bye.aarch64-darwin"
"release-checks" <- Will be dropped
]
into
[
{ name = "hello"; platform = "aarch64-linux"; }
{ name = "hello"; platform = "x86_64-linux"; }
{ name = "hello"; platform = "aarch64-darwin"; }
{ name = "hello"; platform = "x86_64-darwin"; }
{ name = "bye"; platform = "aarch64-darwin"; }
{ name = "bye"; platform = "x86_64-darwin"; }
]
*/
convertToPackagePlatformAttrs =
packagePlatformPaths:
builtins.filter (x: x != null) (builtins.map convertToPackagePlatformAttr packagePlatformPaths);
/*
Converts a list of `packagePlatformPath`s directly to a list of (unique) package names
Turns
[
"hello.aarch64-linux"
"hello.x86_64-linux"
"hello.aarch64-darwin"
"hello.x86_64-darwin"
"bye.x86_64-darwin"
"bye.aarch64-darwin"
]
into
[
"hello"
"bye"
]
*/
extractPackageNames =
packageSystemPaths:
builtins.attrNames (
builtins.removeAttrs (builtins.groupBy (
packageSystemPath: (_processSystemPath packageSystemPath).name
) packageSystemPaths) [ "" ]
);
packagePlatformPaths:
let
packagePlatformAttrs = convertToPackagePlatformAttrs (uniqueStrings packagePlatformPaths);
in
uniqueStrings (builtins.map (p: p.name) packagePlatformAttrs);
# Computes a diff between two attrs
# {
# added: [ <keys only in the second object> ],
# removed: [ <keys only in the first object> ],
# changed: [ <keys with different values between the two objects> ],
# }
#
/*
Computes the key difference between two attrs
{
added: [ <keys only in the second object> ],
removed: [ <keys only in the first object> ],
changed: [ <keys with different values between the two objects> ],
}
*/
diff =
let
filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
@@ -69,43 +115,78 @@ rec {
) old;
};
/*
Group a list of `packagePlatformAttr`s by platforms
Turns
[
{ name = "hello"; platform = "aarch64-linux"; }
{ name = "hello"; platform = "x86_64-linux"; }
{ name = "hello"; platform = "aarch64-darwin"; }
{ name = "hello"; platform = "x86_64-darwin"; }
{ name = "bye"; platform = "aarch64-darwin"; }
{ name = "bye"; platform = "x86_64-darwin"; }
]
into
{
aarch64-linux = [ "hello" ];
x86_64-linux = [ "hello" ];
aarch64-darwin = [ "hello" "bye" ];
x86_64-darwin = [ "hello" "bye" ];
}
*/
groupByPlatform =
packagePlatformAttrs:
let
packagePlatformAttrsByPlatform = builtins.groupBy (p: p.platform) packagePlatformAttrs;
extractPackageNames = map (p: p.name);
in
lib.mapAttrs (_: extractPackageNames) packagePlatformAttrsByPlatform;
# Turns
# [
# "hello.aarch64-linux"
# "hello.x86_64-linux"
# "hello.aarch64-darwin"
# "hello.x86_64-darwin"
# "bye.x86_64-darwin"
# "bye.aarch64-darwin"
# { name = "hello"; platform = "aarch64-linux"; }
# { name = "hello"; platform = "x86_64-linux"; }
# { name = "hello"; platform = "aarch64-darwin"; }
# { name = "hello"; platform = "x86_64-darwin"; }
# { name = "bye"; platform = "aarch64-darwin"; }
# { name = "bye"; platform = "x86_64-darwin"; }
# ]
#
# into
#
# {
# linux = [
# "hello"
# ];
# darwin = [
# "hello"
# "bye"
# ];
# linux = [ "hello" ];
# darwin = [ "hello" "bye" ];
# }
groupByKernel =
systemPaths:
packagePlatformAttrs:
let
systemPaths' = builtins.map _processSystemPath systemPaths;
filterKernel =
kernel:
builtins.attrNames (
builtins.groupBy (systemPath: systemPath.name) (
builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths'
builtins.groupBy (p: p.name) (
builtins.filter (p: lib.hasSuffix kernel p.platform) packagePlatformAttrs
)
);
in
lib.genAttrs [ "linux" "darwin" ] filterKernel;
getLabels = lib.mapAttrs (
/*
Maps an attrs of `kernel - rebuild counts` mappings to a list of labels
Turns
{
linux = 56;
darwin = 8;
}
into
[
"10.rebuild-darwin: 1-10"
"10.rebuild-linux: 11-100"
]
*/
getLabels = lib.mapAttrsToList (
kernel: rebuildCount:
let
number =