top-level/release-outpaths: move to ci/eval & refactor (#439221)

# This expression will, as efficiently as possible, dump a

# *superset* of all attrpaths of derivations which might be

# part of a release on *any* platform.

#

# This expression runs single-threaded under all current Nix

# implementations, but much faster and with much less memory

# used than ./outpaths.nix itself.

#

# Once you have the list of attrnames you can split it up into

# $NUM_CORES batches and evaluate the outpaths separately for each

# batch, in parallel.

#

# To dump the attrnames:

#

#   nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names

#

{

  lib ? import (path + "/lib"),

  trace ? false,

  path ? ./../..,

}:

let

  # TODO: Use mapAttrsToListRecursiveCond when this PR lands:

  # https://github.com/NixOS/nixpkgs/pull/395160

  justAttrNames =

    path: value:

    let

      result =

        if path == [ "AAAAAASomeThingsFailToEvaluate" ] || !(lib.isAttrs value) then

          [ ]

        else if lib.isDerivation value then

          [ path ]

        else

          lib.pipe value [

            (lib.mapAttrsToList (

              name: value:

              lib.addErrorContext "while evaluating package set attribute path '${

                lib.showAttrPath (path ++ [ name ])

              }'" (justAttrNames (path ++ [ name ]) value)

            ))

            lib.concatLists

          ];

    in

    lib.traceIf trace "** ${lib.showAttrPath path}" result;

  outpaths = import ./outpaths.nix {

    inherit path;

    attrNamesOnly = true;

  };

  paths = [

    # Some of the following are based on variants, which are disabled with `attrNamesOnly = true`.

    # Until these have been removed from release.nix / hydra, we manually add them to the list.

    [

      "pkgsLLVM"

      "stdenv"

    ]

    [

      "pkgsArocc"

      "stdenv"

    ]

    [

      "pkgsZig"

      "stdenv"

    ]

    [

      "pkgsStatic"

      "stdenv"

    ]

    [

      "pkgsMusl"

      "stdenv"

    ]

  ]

  ++ justAttrNames [ ] outpaths;

  names = map lib.showAttrPath paths;

in

{

  inherit paths names;

}

# This file works in tandem with ../../ci/eval/default.nix

# It turns ./release-outpaths.nix into chunks of a fixed size

# This turns ./outpaths.nix into chunks of a fixed size.

{

  lib ? import ../../lib,

  path ? ../..,

  attrpathFile,

  chunkSize,

  myChunk,

  checkMeta,

  includeBroken,

  systems,

}:

  attrpaths = lib.importJSON attrpathFile;

  myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths;

  unfiltered = import ./release-outpaths.nix {

  unfiltered = import ./outpaths.nix {

    inherit path;

    inherit checkMeta includeBroken systems;

    inherit includeBroken systems;

  };

  # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths

          ) rebuildsByKernel

          # Set the "11.by: package-maintainer" label to whether all packages directly

          # changed are maintained by the PR's author.

          # (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88)

          // {

            "11.by: package-maintainer" =

              maintainers ? ${githubAuthorId}

{

  lib,

}:

# Almost directly vendored from https://github.com/NixOS/ofborg/blob/5a4e743f192fb151915fcbe8789922fa401ecf48/ofborg/src/maintainers.nix

{

  changedattrs,

  changedpathsjson,

      root = ../..;

      fileset = unions (

        map (lib.path.append ../..) [

          ".version"

          "ci/supportedSystems.json"

          "ci/eval/attrpaths.nix"

          "ci/eval/chunk.nix"

          "ci/eval/outpaths.nix"

          "default.nix"

          "doc"

          "lib"

          "modules"

          "nixos"

          "pkgs"

          ".version"

          "ci/supportedSystems.json"

        ]

      );

    };

        export GC_INITIAL_HEAP_SIZE=4g

        command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \

          nix-instantiate --eval --strict --json --show-trace \

            "$src/pkgs/top-level/release-attrpaths-superset.nix" \

            "$src/ci/eval/attrpaths.nix" \

            -A paths \

            -I "$src" \

            --option restrict-eval true \

      attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",

      # The number of attributes per chunk, see ./README.md for more info.

      chunkSize ? 5000,

      checkMeta ? true,

      # Don't try to eval packages marked as broken.

      includeBroken ? false,

        set +e

        command time -o "$outputDir/timestats/$myChunk" \

          -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \

          nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \

          nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \

          --eval-system "$system" \

          --option restrict-eval true \

          --option allow-import-from-derivation false \

          --arg myChunk "$myChunk" \

          --arg attrpathFile "${attrpathFile}" \

          --arg systems "[ \"$system\" ]" \

          --arg checkMeta ${lib.boolToString checkMeta} \

          --arg includeBroken ${lib.boolToString includeBroken} \

          -I ${nixpkgs} \

          -I ${attrpathFile} \