ci/eval/compare/maintainers: first-class by-name support (#452566)

  combinedDir,

  touchedFilesJson,

  githubAuthorId,

  byName ? false,

}:

let

  # Usually we expect a derivation, but when evaluating in multiple separate steps, we pass

    changedattrs = lib.attrNames (lib.groupBy (a: a.name) changedPackagePlatformAttrs);

    changedpathsjson = touchedFilesJson;

    removedattrs = lib.attrNames (lib.groupBy (a: a.name) removedPackagePlatformAttrs);

    inherit byName;

  };

in

runCommand "compare"

  changedattrs,

  changedpathsjson,

  removedattrs,

  byName ? false,

}:

let

  pkgs = import ../../.. {

    system = "x86_64-linux";

    config = { };

    overlays = [ ];

  };

  pkgs = import ../../.. { system = "x86_64-linux"; };

  changedpaths = builtins.fromJSON (builtins.readFile changedpathsjson);

  changedpaths = lib.importJSON changedpathsjson;

  anyMatchingFile =

    filename: builtins.any (changed: lib.strings.hasSuffix changed filename) changedpaths;

  # Extract attributes that changed from by-name paths.

  # This allows pinging reviewers for pure refactors.

  touchedattrs = lib.pipe changedpaths [

    (lib.filter (changed: lib.hasPrefix "pkgs/by-name/" changed))

    (map (lib.splitString "/"))

    (map (path: lib.elemAt path 3))

    lib.unique

  ];

  anyMatchingFile = filename: lib.any (lib.hasPrefix filename) changedpaths;

  anyMatchingFiles = files: lib.any anyMatchingFile files;

  anyMatchingFiles = files: builtins.any anyMatchingFile files;

  sharded = name: "${lib.substring 0 2 name}/${name}";

  attrsWithMaintainers = lib.pipe (changedattrs ++ removedattrs) [

  attrsWithMaintainers = lib.pipe (changedattrs ++ removedattrs ++ touchedattrs) [

    # An attribute can appear in changed/removed *and* touched

    lib.unique

    (map (

      name:

      let

        path = lib.splitString "." name;

        # Some packages might be reported as changed on a different platform, but

        # not even have an attribute on the platform the maintainers are requested on.

        # Fallback to `null` for these to filter them out below.

        package = lib.attrByPath (lib.splitString "." name) null pkgs;

        package = lib.attrByPath path null pkgs;

      in

      {

        inherit name package;

        # Adds all files in by-name to each package, no matter whether they are discoverable

        # via meta attributes below. For example, this allows pinging maintainers for

        # updates to .json files.

        # TODO: Support by-name package sets.

        filenames = lib.optional (lib.length path == 1) "pkgs/by-name/${sharded (lib.head path)}/";

        # TODO: Refactor this so we can ping entire teams instead of the individual members.

        # Note that this will require keeping track of GH team IDs in "maintainers/teams.nix".

        maintainers = package.meta.maintainers or [ ];

    # No need to match up packages without maintainers with their files.

    # This also filters out attributes where `packge = null`, which is the

    # case for libintl, for example.

    (builtins.filter (pkg: pkg.maintainers != [ ]))

    (lib.filter (pkg: pkg.maintainers != [ ]))

  ];

  relevantFilenames =

    drv:

    (lib.lists.unique (

      map (pos: lib.strings.removePrefix (toString ../..) pos.file) (

        builtins.filter (x: x != null) [

          ((drv.meta or { }).maintainersPosition or null)

          ((drv.meta or { }).teamsPosition or null)

          (builtins.unsafeGetAttrPos "src" drv)

          # broken because name is always set by stdenv:

          #    # A hack to make `nix-env -qa` and `nix search` ignore broken packages.

          #    # TODO(@oxij): remove this assert when something like NixOS/nix#1771 gets merged into nix.

          #    name = assert validity.handled; name + lib.optionalString

          #(builtins.unsafeGetAttrPos "name" drv)

          (builtins.unsafeGetAttrPos "pname" drv)

          (builtins.unsafeGetAttrPos "version" drv)

    (lib.unique (

      map (pos: lib.removePrefix "${toString ../../..}/" pos.file) (

        lib.filter (x: x != null) [

          (drv.meta.maintainersPosition or null)

          (drv.meta.teamsPosition or null)

          (lib.unsafeGetAttrPos "src" drv)

          (lib.unsafeGetAttrPos "pname" drv)

          (lib.unsafeGetAttrPos "version" drv)

          # Use ".meta.position" for cases when most of the package is

          # defined in a "common" section and the only place where

    ));

  attrsWithFilenames = map (

    pkg: pkg // { filenames = relevantFilenames pkg.package; }

    pkg: pkg // { filenames = pkg.filenames ++ relevantFilenames pkg.package; }

  ) attrsWithMaintainers;

  attrsWithModifiedFiles = builtins.filter (pkg: anyMatchingFiles pkg.filenames) attrsWithFilenames;

  attrsWithModifiedFiles = lib.filter (pkg: anyMatchingFiles pkg.filenames) attrsWithFilenames;

  listToPing = lib.concatMap (

    pkg:

    }) pkg.maintainers

  ) attrsWithModifiedFiles;

  byMaintainer = lib.groupBy (ping: toString ping.${if byName then "github" else "id"}) listToPing;

  byMaintainer = lib.groupBy (ping: toString ping.id) listToPing;

  packagesPerMaintainer = lib.attrsets.mapAttrs (

  packagesPerMaintainer = lib.mapAttrs (

    maintainer: packages: map (pkg: pkg.packageName) packages

  ) byMaintainer;

in

#!/usr/bin/env nix-shell

#!nix-shell -i bash -p git jq

# Outputs a list of maintainers that would be pinged across two nixpkgs revisions.

# Authors:

#  Morgan Jones (@numinit)

#  Tristan Ross (@RossComputerGuy)

set -euo pipefail

if [ $# -lt 2 ]; then

    echo "Usage: $0 <rev-from> <rev-to>" >&2

    exit 1

fi

repo="$(git rev-parse --show-toplevel)"

system="$(nix-instantiate --eval --expr builtins.currentSystem)"

rev1="$(git -C "$repo" rev-parse "$1")"

rev2="$(git -C "$repo" rev-parse "$2")"

echo "Touched files:" >&2

git -C "$repo" diff --name-only "$rev1" "$rev2" \

    | jq --raw-input --slurp 'split("\n")[:-1]' | tee "$TMPDIR/touched-files.json" >&2

# Runs an eval in the given worktree, outputting the path to $TMPDIR/$1.path.

# $1: The revision SHA.

eval_in_worktree() (

    mkdir -p .worktree

    local rev="$1"

    local tree=".worktree/$rev"

    if [ ! -d "$tree" ]; then

        git -C "$repo" worktree add -f -d "$tree" "$rev" >&2

    fi

    cd "$tree"

    local workdir="$TMPDIR/$rev"

    rm -rf "$workdir"

    mkdir -p "$workdir"

    nix-build ci -A eval.attrpathsSuperset -o "$workdir/paths" >&2

    mkdir -p "$workdir/intermediates"

    nix-build ci -A eval.singleSystem \

        --arg evalSystem "$system" \

        --arg attrpathFile "$workdir/paths/paths.json" \

        --arg chunkSize ${CHUNK_SIZE:-10000} \

        -o "$workdir/intermediates/.intermediate-1" >&2

    # eval.combine nix-build needs a directory, not a symlink

    cp -RL "$workdir/intermediates/.intermediate-1" "$workdir/intermediates/intermediate-1"

    chmod -R +w "$workdir/intermediates/intermediate-1"

    rm -rf "$workdir/intermediates/.intermediate-1"

    nix-build ci -A eval.combine \

        --arg resultsDir "$workdir/intermediates" \

        -o "$workdir/result" >&2

)

eval_in_worktree "$rev1" &

pid1=$!

eval_in_worktree "$rev2" &

pid2=$!

wait $pid1

wait $pid2

path1="$TMPDIR/$rev1"

path2="$TMPDIR/$rev2"

# Use the repo this script was executed in to get accurate maintainer info

nix-build "$repo/ci" -A eval.compare \

    --arg beforeResultDir "$path1/result" \

    --arg afterResultDir "$path2/result" \

    --arg touchedFilesJson "$TMPDIR/touched-files.json" \

    --arg byName true \

    -o comparison

echo "Pinged maintainers (check $repo/comparison for more details)" >&2

jq < comparison/maintainers.json