Unverified Commit 8a30b52c authored by Martin Weinelt's avatar Martin Weinelt Committed by GitHub
Browse files

Merge pull request #306095 from K900/you-can-just-leave

Tarball job optimizations
parents 9140927f f7672530
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ let

  trace = if builtins.getEnv "VERBOSE" == "1" then builtins.trace else (x: y: y);

  rel = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" "xbursttools" ];
  rel = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ];

  # Add the ‘recurseForDerivations’ attribute to ensure that
  # nix-instantiate recurses into nested attribute sets.
+27 −36
Original line number Diff line number Diff line
/* Hydra job to build a tarball for Nixpkgs from a Git checkout.  It
   also builds the documentation and tests whether the Nix expressions
   evaluate correctly. */

{ nixpkgs
, officialRelease
, supportedSystems
, pkgs ? import nixpkgs.outPath {}
, nix ? pkgs.nix
, lib-tests ? import ../../lib/tests/release.nix { inherit pkgs; }
@@ -32,54 +27,50 @@ pkgs.releaseTools.sourceTarball {
    echo "git-revision is $(cat .git-revision)"
  '';

  requiredSystemFeatures = [ "big-parallel" ]; # 1 thread but ~36G RAM (!) see #227945
  dontUnpack = true;

  dontBuild = false;

  doCheck = true;

  checkPhase = ''
    set -o pipefail

    export NIX_STATE_DIR=$TMPDIR
    export NIX_PATH=nixpkgs=$TMPDIR/barf.nix
    opts=(--option build-users-group "")
    nix-store --init
    echo "generating packages.json"

    echo "checking eval-release.nix"
    nix-instantiate --eval --strict --show-trace ./maintainers/scripts/eval-release.nix > /dev/null
    (
      echo -n '{"version":2,"packages":'
      NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}'
      echo -n '}'
    ) | sed "s|$src/||g" | jq -c > packages.json

    echo "checking find-tarballs.nix"
    nix-instantiate --readonly-mode --eval --strict --show-trace --json \
       ./maintainers/scripts/find-tarballs.nix \
      --arg expr 'import ./maintainers/scripts/all-tarballs.nix' > $TMPDIR/tarballs.json
    nrUrls=$(jq -r '.[].url' < $TMPDIR/tarballs.json | wc -l)
    echo "found $nrUrls URLs"
    if [ "$nrUrls" -lt 10000 ]; then
      echo "suspiciously low number of URLs"
    # Arbitrary number. The index has ~115k packages as of April 2024.
    if [ $(jq -r '.packages | length' < packages.json) -lt 100000 ]; then
      echo "ERROR: not enough packages in the search index, bailing out!"
      exit 1
    fi

    echo "generating packages.json"
    mkdir -p $out/nix-support
    echo -n '{"version":2,"packages":' > tmp
    nix-env -f . -I nixpkgs=$src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}' "''${opts[@]}" >> tmp
    echo -n '}' >> tmp
    packages=$out/packages.json.br
    < tmp sed "s|$(pwd)/||g" | jq -c | brotli -9 > $packages
    rm tmp

    mkdir -p $out/nix-support
    brotli -9 < packages.json > $packages
    echo "file json-br $packages" >> $out/nix-support/hydra-build-products
  '';

  distPhase = ''
    mkdir -p $out/tarballs
    mkdir ../$releaseName
    cp -prd . ../$releaseName
    (cd .. && tar cfa $out/tarballs/$releaseName.tar.xz $releaseName) || false
    XZ_OPT="-T0" tar \
      --create \
      --xz \
      --file=$out/tarballs/$releaseName.tar.xz \
      --absolute-names \
      --transform="s|^$src|$releaseName|g" \
      --transform="s|^$(pwd)|$releaseName|g" \
      --owner=0 \
      --group=0 \
      --numeric-owner \
      --format=gnu \
      --sort=name \
      --mtime="@$SOURCE_DATE_EPOCH" \
      --mode=ug+w \
      $src $(pwd)/{.version-suffix,.git-revision}
  '';

  meta = {
    maintainers = [ ];
  };
}
+1 −1
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ in
{

  tarball = import ./make-tarball.nix {
    inherit nixpkgs supportedSystems;
    inherit nixpkgs;
    officialRelease = false;
  };

+1 −1
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ let
  ] (arch: elem "${arch}-darwin" supportedSystems);

  nonPackageJobs =
    { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease supportedSystems; };
    { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease; };

      release-checks = import ./nixpkgs-basic-release-checks.nix { inherit pkgs nixpkgs supportedSystems; };