Parallel GH actions workflow for Nixpkgs eval (#356023)

name: Eval

on: pull_request_target

permissions:

  contents: read

jobs:

  attrs:

    name: Attributes

    runs-on: ubuntu-latest

    outputs:

      mergedSha: ${{ steps.merged.outputs.mergedSha }}

      systems: ${{ steps.systems.outputs.systems }}

    steps:

      # Important: Because of `pull_request_target`, this doesn't check out the PR,

      # but rather the base branch of the PR, which is needed so we don't run untrusted code

      - name: Check out the ci directory of the base branch

        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

        with:

          path: base

          sparse-checkout: ci

      - name: Check if the PR can be merged and get the test merge commit

        id: merged

        env:

          GH_TOKEN: ${{ github.token }}

        run: |

          if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then

            echo "Checking the merge commit $mergedSha"

            echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"

          else

            # Skipping so that no notifications are sent

            echo "Skipping the rest..."

          fi

          rm -rf base

      - name: Check out the PR at the test merge commit

        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

        # Add this to _all_ subsequent steps to skip them

        if: steps.merged.outputs.mergedSha

        with:

          ref: ${{ env.mergedSha }}

          path: nixpkgs

      - name: Install Nix

        uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30

        if: steps.merged.outputs.mergedSha

      - name: Evaluate the list of all attributes and get the systems matrix

        id: systems

        if: steps.merged.outputs.mergedSha

        run: |

          nix-build nixpkgs/ci -A eval.attrpathsSuperset

          echo "systems=$(<result/systems.json)" >> "$GITHUB_OUTPUT"

      - name: Upload the list of all attributes

        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

        if: steps.merged.outputs.mergedSha

        with:

          name: paths

          path: result/*

  outpaths:

    name: Outpaths

    runs-on: ubuntu-latest

    needs: attrs

    # Skip this and future steps if the PR can't be merged

    if: needs.attrs.outputs.mergedSha

    strategy:

      matrix:

        system: ${{ fromJSON(needs.attrs.outputs.systems) }}

    steps:

      - name: Download the list of all attributes

        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8

        with:

          name: paths

          path: paths

      - name: Check out the PR at the test merge commit

        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

        with:

          ref: ${{ needs.attrs.outputs.mergedSha }}

          path: nixpkgs

      - name: Install Nix

        uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30

      - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes

        run: |

          nix-build nixpkgs/ci -A eval.singleSystem \

            --argstr evalSystem ${{ matrix.system }} \

            --arg attrpathFile ./paths/paths.json \

            --arg chunkSize 10000

          # If it uses too much memory, slightly decrease chunkSize

      - name: Upload the output paths and eval stats

        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

        if: needs.attrs.outputs.mergedSha

        with:

          name: intermediate-${{ matrix.system }}

          path: result/*

  process:

    name: Process

    runs-on: ubuntu-latest

    needs: outpaths

    steps:

      - name: Download output paths and eval stats for all systems

        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8

        with:

          pattern: intermediate-*

          path: intermediate

      - name: Check out the PR at the test merge commit

        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

        with:

          ref: ${{ needs.attrs.outputs.mergedSha }}

          path: nixpkgs

      - name: Install Nix

        uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30

      - name: Combine all output paths and eval stats

        run: |

          nix-build nixpkgs/ci -A eval.combine \

            --arg resultsDir ./intermediate

      - name: Upload the combined results

        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

        with:

          name: result

          path: result/*

      # TODO: Run this workflow also on `push` (on at least the main development branches)

      # Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)

      # to have completed the eval, then use

      # gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>

      # and follow it to the artifact results, where you can then download the outpaths.json from the base branch

      # That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers

  inherit pkgs;

  requestReviews = pkgs.callPackage ./request-reviews { };

  codeownersValidator = pkgs.callPackage ./codeowners-validator { };

  eval = pkgs.callPackage ./eval { };

}

# Nixpkgs CI evaluation

The code in this directory is used by the [eval.yml](../../.github/workflows/eval.yml) GitHub Actions workflow to evaluate the majority of Nixpkgs for all PRs, effectively making sure that when the development branches are processed by Hydra, no evaluation failures are encountered.

Furthermore it also allows local evaluation using

```

nix-build ci -A eval.full \

  --max-jobs 4

  --cores 2

  --arg chunkSize 10000

```

- `--max-jobs`: The maximum number of derivations to run at the same time. Only each [supported system](../supportedSystems.nix) gets a separate derivation, so it doesn't make sense to set this higher than that number.

- `--cores`: The number of cores to use for each job. Recommended to set this to the amount of cores on your system divided by `--max-jobs`.

- `chunkSize`: The number of attributes that are evaluated simultaneously on a single core. Lowering this decreases memory usage at the cost of increased evaluation time. If this is too high, there won't be enough chunks to process them in parallel, and will also increase evaluation time.

A good default is to set `chunkSize` to 10000, which leads to about 3.6GB max memory usage per core, so suitable for fully utilising machines with 4 cores and 16GB memory, 8 cores and 32GB memory or 16 cores and 64GB memory.

Note that 16GB memory is the recommended minimum, while with less than 8GB memory evaluation time suffers greatly.

{

  lib,

  runCommand,

  writeShellScript,

  linkFarm,

  time,

  procps,

  nix,

  jq,

  sta,

}:

let

  nixpkgs =

    with lib.fileset;

    toSource {

      root = ../..;

      fileset = unions (

        map (lib.path.append ../..) [

          "default.nix"

          "doc"

          "lib"

          "maintainers"

          "nixos"

          "pkgs"

          ".version"

          "ci/supportedSystems.nix"

        ]

      );

    };

  supportedSystems = import ../supportedSystems.nix;

  attrpathsSuperset =

    runCommand "attrpaths-superset.json"

      {

        src = nixpkgs;

        nativeBuildInputs = [

          nix

          time

        ];

        env.supportedSystems = builtins.toJSON supportedSystems;

        passAsFile = [ "supportedSystems" ];

      }

      ''

        export NIX_STATE_DIR=$(mktemp -d)

        mkdir $out

        export GC_INITIAL_HEAP_SIZE=4g

        command time -v \

          nix-instantiate --eval --strict --json --show-trace \

          $src/pkgs/top-level/release-attrpaths-superset.nix -A paths \

          --arg enableWarnings false > $out/paths.json

        mv "$supportedSystemsPath" $out/systems.json

      '';

  singleSystem =

    {

      # The system to evaluate.

      # Note that this is intentionally not called `system`,

      # because `--argstr system` would only be passed to the ci/default.nix file!

      evalSystem,

      # The path to the `paths.json` file from `attrpathsSuperset`

      attrpathFile,

      # The number of attributes per chunk, see ./README.md for more info.

      chunkSize,

      checkMeta ? true,

      includeBroken ? true,

      # Whether to just evaluate a single chunk for quick testing

      quickTest ? false,

    }:

    let

      singleChunk = writeShellScript "single-chunk" ''

        set -euo pipefail

        chunkSize=$1

        myChunk=$2

        system=$3

        outputDir=$4

        export NIX_SHOW_STATS=1

        export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk"

        echo "Chunk $myChunk on $system start"

        set +e

        command time -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \

          nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \

          --query --available \

          --no-name --attr-path --out-path \

          --show-trace \

          --arg chunkSize "$chunkSize" \

          --arg myChunk "$myChunk" \

          --arg attrpathFile "${attrpathFile}" \

          --arg systems "[ \"$system\" ]" \

          --arg checkMeta ${lib.boolToString checkMeta} \

          --arg includeBroken ${lib.boolToString includeBroken} \

          > "$outputDir/result/$myChunk"

        exitCode=$?

        set -e

        if (( exitCode != 0 )); then

          echo "Evaluation failed with exit code $exitCode"

          # This immediately halts all xargs processes

          kill $PPID

        fi

      '';

    in

    runCommand "nixpkgs-eval-${evalSystem}"

      {

        nativeBuildInputs = [

          nix

          time

          procps

          jq

        ];

        env = {

          inherit evalSystem chunkSize;

        };

      }

      ''

        export NIX_STATE_DIR=$(mktemp -d)

        nix-store --init

        echo "System: $evalSystem"

        cores=$NIX_BUILD_CORES

        echo "Cores: $cores"

        attrCount=$(jq length "${attrpathFile}")

        echo "Attribute count: $attrCount"

        echo "Chunk size: $chunkSize"

        # Same as `attrCount / chunkSize` but rounded up

        chunkCount=$(( (attrCount - 1) / chunkSize + 1 ))

        echo "Chunk count: $chunkCount"

        mkdir $out

        # Record and print stats on free memory and swap in the background

        (

          while true; do

            availMemory=$(free -b | grep Mem | awk '{print $7}')

            freeSwap=$(free -b | grep Swap | awk '{print $4}')

            echo "Available memory: $(( availMemory / 1024 / 1024 )) MiB, free swap: $(( freeSwap / 1024 / 1024 )) MiB"

            if [[ ! -f "$out/min-avail-memory" ]] || (( availMemory < $(<$out/min-avail-memory) )); then

              echo "$availMemory" > $out/min-avail-memory

            fi

            if [[ ! -f $out/min-free-swap ]] || (( availMemory < $(<$out/min-free-swap) )); then

              echo "$freeSwap" > $out/min-free-swap

            fi

            sleep 4

          done

        ) &

        seq_end=$(( chunkCount - 1 ))

        ${lib.optionalString quickTest ''

          seq_end=0

        ''}

        chunkOutputDir=$(mktemp -d)

        mkdir "$chunkOutputDir"/{result,stats}

        seq -w 0 "$seq_end" |

          command time -f "%e" -o "$out/total-time" \

          xargs -I{} -P"$cores" \

          ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir"

        if (( chunkSize * chunkCount != attrCount )); then

          # A final incomplete chunk would mess up the stats, don't include it

          rm "$chunkOutputDir"/stats/"$seq_end"

        fi

        # Make sure the glob doesn't break when there's no files

        shopt -s nullglob

        cat "$chunkOutputDir"/result/* > $out/paths

        cat "$chunkOutputDir"/stats/* > $out/stats.jsonstream

      '';

  combine =

    {

      resultsDir,

    }:

    runCommand "combined-result"

      {

        nativeBuildInputs = [

          jq

          sta

        ];

      }

      ''

        mkdir -p $out

        # Transform output paths to JSON

        cat ${resultsDir}/*/paths |

          jq --sort-keys --raw-input --slurp '

            split("\n") |

            map(select(. != "") | split(" ") | map(select(. != ""))) |

            map(

              {

                key: .[0],

                value: .[1] | split(";") | map(split("=") |

                  if length == 1 then

                    { key: "out", value: .[0] }

                  else

                    { key: .[0], value: .[1] }

                  end) | from_entries}

            ) | from_entries

          ' > $out/outpaths.json

        # Computes min, mean, error, etc. for a list of values and outputs a JSON from that

        statistics() {

          local stat=$1

          sta --transpose |

            jq --raw-input --argjson stat "$stat" -n '

              [

                inputs |

                  split("\t") |

                  { key: .[0], value: (.[1] | fromjson) }

              ] |

                from_entries |

                {

                  key: ($stat | join(".")),

                  value: .

                }'

        }

        # Gets all available number stats (without .sizes because those are constant and not interesting)

        readarray -t stats < <(jq -cs '.[0] | del(.sizes) | paths(type == "number")' ${resultsDir}/*/stats.jsonstream)

        # Combines the statistics from all evaluations

        {

          echo "{ \"key\": \"minAvailMemory\", \"value\": $(cat ${resultsDir}/*/min-avail-memory | sta --brief --min) }"

          echo "{ \"key\": \"minFreeSwap\", \"value\": $(cat ${resultsDir}/*/min-free-swap | sta --brief --min) }"

          cat ${resultsDir}/*/total-time | statistics '["totalTime"]'

          for stat in "''${stats[@]}"; do

            cat ${resultsDir}/*/stats.jsonstream |

              jq --argjson stat "$stat" 'getpath($stat)' |

              statistics "$stat"

          done

        } |

          jq -s from_entries > $out/stats.json

      '';

  full =

    {

      # Whether to evaluate just a single system, by default all are evaluated

      evalSystem ? if quickTest then "x86_64-linux" else null,

      # The number of attributes per chunk, see ./README.md for more info.

      chunkSize,

      quickTest ? false,

    }:

    let

      systems = if evalSystem == null then supportedSystems else [ evalSystem ];

      results = linkFarm "results" (

        map (evalSystem: {

          name = evalSystem;

          path = singleSystem {

            inherit quickTest evalSystem chunkSize;

            attrpathFile = attrpathsSuperset + "/paths.json";

          };

        }) systems

      );

    in

    combine {

      resultsDir = results;

    };

in

{

  inherit

    attrpathsSuperset

    singleSystem

    combine

    # The above three are used by separate VMs in a GitHub workflow,

    # while the below is intended for testing on a single local machine

    full

    ;

}

[

  "aarch64-linux"

  "aarch64-darwin"

  "x86_64-linux"

  "x86_64-darwin"

]