Commit 80e01114 authored by Silvan Mosberger's avatar Silvan Mosberger
Browse files

ci/eval/compare: Improve performance and avoid large stacks 

Various improvements such as:
1. Avoiding deduplications when there can't be any duplicates
2. Avoiding O(n^2) deduplications
3. Using builtins.any to avoid list allocations
4. Using builtins.concatMap instead of lib.flatten when it's known that there's only one level of nesting
5. Using builtins.groupBy instead of folding with an accumulator

In particular 5. should fix CI exceeding the stack size on staging: https://github.com/NixOS/nixpkgs/actions/runs/12989244871/job/36240781244?pr=377253

While 2. in particular should make CI a lot faster.
parent 0fe2e3f0

ci/eval/compare/default.nix

+2 −3
Original line number Diff line number Diff line
@@ -69,7 +69,6 @@ let 
    groupByPlatform

    extractPackageNames

    getLabels

    uniqueStrings

    ;



  getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json");
@@ -81,7 +80,7 @@ let 
  # - values: lists of `packagePlatformPath`s

  diffAttrs = diff beforeAttrs afterAttrs;



  rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);

  rebuilds = diffAttrs.added ++ diffAttrs.changed;

  rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;



  changed-paths =
@@ -110,7 +109,7 @@ let 
    );



  maintainers = import ./maintainers.nix {

    changedattrs = lib.unique (map (a: a.packagePath) rebuildsPackagePlatformAttrs);

    changedattrs = lib.attrNames (lib.groupBy (a: a.name) rebuildsPackagePlatformAttrs);

    changedpathsjson = touchedFilesJson;

  };

in

ci/eval/compare/maintainers.nix

+14 −29
Original line number Diff line number Diff line
@@ -11,17 +11,13 @@ let 
  changedpaths = builtins.fromJSON (builtins.readFile changedpathsjson);



  anyMatchingFile =

    filename:

    let

      matching = builtins.filter (changed: lib.strings.hasSuffix changed filename) changedpaths;

    in

    (builtins.length matching) > 0;

    filename: builtins.any (changed: lib.strings.hasSuffix changed filename) changedpaths;



  anyMatchingFiles = files: (builtins.length (builtins.filter anyMatchingFile files)) > 0;

  anyMatchingFiles = files: builtins.any anyMatchingFile files;



  enrichedAttrs = builtins.map (path: {

    path = path;

    name = builtins.concatStringsSep "." path;

  enrichedAttrs = builtins.map (name: {

    path = lib.splitString "." name;

    name = name;

  }) changedattrs;



  validPackageAttributes = builtins.filter (
@@ -80,27 +76,16 @@ let 


  attrsWithModifiedFiles = builtins.filter (pkg: anyMatchingFiles pkg.filenames) attrsWithFilenames;



  listToPing = lib.lists.flatten (

    builtins.map (

  listToPing = lib.concatMap (

    pkg:

    builtins.map (maintainer: {

      id = maintainer.githubId;

      packageName = pkg.name;

      dueToFiles = pkg.filenames;

    }) pkg.maintainers

    ) attrsWithModifiedFiles

  );



  byMaintainer = lib.lists.foldr (

    ping: collector:

    collector

    // {

      "${toString ping.id}" = [

        { inherit (ping) packageName dueToFiles; }

      ] ++ (collector."${toString ping.id}" or [ ]);

    }

  ) { } listToPing;

  ) attrsWithModifiedFiles;



  byMaintainer = lib.groupBy (ping: toString ping.id) listToPing;



  packagesPerMaintainer = lib.attrsets.mapAttrs (

    maintainer: packages: builtins.map (pkg: pkg.packageName) packages