bazel 7.1 -> 7.3 (#338264)

{ stdenv

{

  stdenv,

  # nix tooling and utilities

, callPackage

, lib

, fetchurl

, makeWrapper

, writeTextFile

, substituteAll

, writeShellApplication

, makeBinaryWrapper

  lib,

  fetchurl,

  makeWrapper,

  writeTextFile,

  substituteAll,

  writeShellApplication,

  makeBinaryWrapper,

  autoPatchelfHook,

  buildFHSEnv,

  # this package (through the fixpoint glass)

, bazel_self

  # TODO probably still need for tests at some point

  bazel_self,

  # native build inputs

, runtimeShell

, zip

, unzip

, bash

, coreutils

, which

, gawk

, gnused

, gnutar

, gnugrep

, gzip

, findutils

, diffutils

, gnupatch

, file

, installShellFiles

, lndir

, python3

  runtimeShell,

  zip,

  unzip,

  bash,

  coreutils,

  which,

  gawk,

  gnused,

  gnutar,

  gnugrep,

  gzip,

  findutils,

  diffutils,

  gnupatch,

  file,

  installShellFiles,

  lndir,

  python3,

  # Apple dependencies

, cctools

, libcxx

, sigtool

, CoreFoundation

, CoreServices

, Foundation

, IOKit

  cctools,

  libcxx,

  libtool,

  sigtool,

  CoreFoundation,

  CoreServices,

  Foundation,

  IOKit,

  # Allow to independently override the jdks used to build and run respectively

, buildJdk

, runJdk

  buildJdk,

  runJdk,

  # Always assume all markers valid (this is needed because we remove markers; they are non-deterministic).

  # Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers).

, enableNixHacks ? false

, version ? "7.1.2"

  enableNixHacks ? false,

  version ? "7.3.1",

}:

let

  src = fetchurl {

    url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";

    hash = "sha256-nPbtIxnIFpGdlwFe720MWULNGu1I4DxzuggV2VPtYas=";

  };

  # Use builtins.fetchurl to avoid IFD, in particular on hydra

  #lockfile = builtins.fetchurl {

  #  url = "https://raw.githubusercontent.com/bazelbuild/bazel/release-${version}/MODULE.bazel.lock";

  #  sha256 = "sha256-5xPpCeWVKVp1s4RVce/GoW2+fH8vniz5G1MNI4uezpc=";

  #};

  # Use a local copy of the above lockfile to make ofborg happy.

  lockfile = ./MODULE.bazel.lock;

  # Two-in-one format

  distDir = repoCache;

  repoCache = callPackage ./bazel-repository-cache.nix {

    inherit lockfile;

    # We use the release tarball that already has everything bundled so we

    # should not need any extra external deps. But our nonprebuilt java

    # toolchains hack needs just one non bundled dep.

    requiredDepNamePredicate = name:

      null != builtins.match "rules_java~.*~toolchains~remote_java_tools" name;

    hash = "sha256-8FAfkMn8dM1pM9vcWeF7jWJy1sCfi448QomFxYlxR8c=";

  };

  defaultShellUtils =

      unzip

      which

      zip

      makeWrapper

    ];

  # Bootstrap an existing Bazel so we can vendor deps with vendor mode

  bazelBootstrap = stdenv.mkDerivation rec {

    name = "bazelBootstrap";

    src =

      if stdenv.hostPlatform.system == "x86_64-linux" then

        fetchurl {

          url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel_nojdk-${version}-linux-x86_64";

          hash = "sha256-05fHtz47OilpOVYawB17VRVEDpycfYTIHBmwYCOyPjI=";

        }

      else if stdenv.hostPlatform.system == "aarch64-linux" then

        fetchurl {

          url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel_nojdk-${version}-linux-arm64";

          hash = "sha256-olrlIia/oXWleXp12E+LGXv+F1m4/S4jj/t7p2/xGdM=";

        }

      else if stdenv.hostPlatform.system == "x86_64-darwin" then

        fetchurl {

          url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-darwin-x86_64";

          hash = "sha256-LraN6MSVJQ3NzkyeLl5LvGxf+VNDJiVo/dVJIkyF1jU=";

        }

      else

        fetchurl {

          # stdenv.hostPlatform.system == "aarch64-darwin"

          url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-darwin-arm64";

          hash = "sha256-mB+CpHC60TSTIrb1HJxv+gqikdqxAU+sQRVDwS5mHf8=";

        };

    nativeBuildInputs = defaultShellUtils;

    buildInputs = [

      stdenv.cc.cc

    ] ++ lib.optional (!stdenv.hostPlatform.isDarwin) autoPatchelfHook;

    dontUnpack = true;

    dontPatch = true;

    dontBuild = true;

    dontStrip = true;

    installPhase = ''

      runHook preInstall

      mkdir -p $out/bin

      install -Dm755 $src $out/bin/bazel

      runHook postInstall

    '';

    postFixup = ''

      wrapProgram $out/bin/bazel \

        --prefix PATH : ${lib.makeBinPath nativeBuildInputs}

    '';

    meta.sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];

  };

  bazelFhs = buildFHSEnv {

    name = "bazel";

    targetPkgs = _: [ bazelBootstrap ];

    runScript = "bazel";

  };

  # A FOD that vendors the Bazel dependencies using Bazel's new vendor mode.

  # See https://bazel.build/versions/7.3.0/external/vendor for details.

  # Note that it may be possible to vendor less than the full set of deps in

  # the future, as this is approximately 16GB.

  bazelDeps =

    let

      bazelForDeps = if stdenv.hostPlatform.isDarwin then bazelBootstrap else bazelFhs;

    in

    stdenv.mkDerivation {

      name = "bazelDeps";

      inherit src version;

      sourceRoot = ".";

      patches = [

        # The repo rule that creates a manifest of the bazel source for testing

        # the cli is not reproducible. This patch ensures that it is by sorting

        # the results in the repo rule rather than the downstream genrule.

        ./test_source_sort.patch

      ];

      patchFlags = [

        "--no-backup-if-mismatch"

        "-p1"

      ];

      nativeBuildInputs = [

        unzip

        runJdk

        bazelForDeps

      ] ++ lib.optional (stdenv.hostPlatform.isDarwin) libtool;

      configurePhase = ''

        runHook preConfigure

        mkdir bazel_src

        shopt -s dotglob extglob

        mv !(bazel_src) bazel_src

        mkdir vendor_dir

        runHook postConfigure

      '';

      dontFixup = true;

      buildPhase = ''

        runHook preBuild

        export HOME=$(mktemp -d)

        (cd bazel_src; ${bazelForDeps}/bin/bazel --server_javabase=${runJdk} mod deps --curses=no;

        ${bazelForDeps}/bin/bazel --server_javabase=${runJdk} vendor src:bazel_nojdk \

        --curses=no \

        --vendor_dir ../vendor_dir \

        --verbose_failures \

        --experimental_strict_java_deps=off \

        --strict_proto_deps=off \

        --tool_java_runtime_version=local_jdk_21 \

        --java_runtime_version=local_jdk_21 \

        --tool_java_language_version=21 \

        --java_language_version=21)

        # Some post-fetch fixup is necessary, because the deps come with some

        # baggage that is not reproducible. Luckily, this baggage does not factor

        # into the final product, so removing it is enough.

        # the GOCACHE is poisonous!

        rm -rf vendor_dir/gazelle~~non_module_deps~bazel_gazelle_go_repository_cache/gocache

        # as is the go versions file (changes when new versions show up)

        rm -f vendor_dir/rules_go~~go_sdk~go_default_sdk/versions.json

        # and so are .pyc files

        find vendor_dir -name "*.pyc" -type f -delete

        # bazel-external is auto-generated and should be removed

        # see https://bazel.build/external/vendor#vendor-symlinks for more details

        rm vendor_dir/bazel-external

        runHook postBuild

      '';

      installPhase = ''

        mkdir -p $out/vendor_dir

        cp -r --reflink=auto vendor_dir/* $out/vendor_dir

      '';

      outputHashMode = "recursive";

      outputHash =

        if stdenv.hostPlatform.system == "x86_64-linux" then

          "sha256-II5R2YjaIejcO4Topdcz1H268eplYsYrW2oLJHKEkYw="

        else if stdenv.hostPlatform.system == "aarch64-linux" then

          "sha256-n8RMKf8OxJsEkcxLe7xZgMu9RyeU58NESFF9F0nLNC4="

        else if stdenv.hostPlatform.system == "aarch64-darwin" then

          "sha256-E6j31Sl+aGs6+Xdx+c0Xi6ryfYZ/ms5/HzIyc3QpMHY="

        else

          # x86_64-darwin

          "sha256-VVuNGY4+SFDhcv9iEo8JToYPzqk9NQCrYlLhhae89MM=";

      outputHashAlgo = "sha256";

    };

  defaultShellPath = lib.makeBinPath defaultShellUtils;

  bashWithDefaultShellUtilsSh = writeShellApplication {

  inherit version src;

  inherit sourceRoot;

  patches = [

  patches =

    [

      # Remote java toolchains do not work on NixOS because they download binaries,

      # so we need to use the @local_jdk//:jdk

      # It could in theory be done by registering @local_jdk//:all toolchains,

          -e 's!/bin/bash!${bashWithDefaultShellUtils}/bin/bash!g' \

          -e 's!shasum -a 256!sha256sum!g'

        # Augment bundled repository_cache with our extra paths

        ${lndir}/bin/lndir ${repoCache}/content_addressable \

          $PWD/derived/repository_cache/content_addressable

        # Add required flags to bazel command line.

        # XXX: It would suit a bazelrc file better, but I found no way to pass it.

        #      It seems that bazel bootstrapping ignores it.

        sedVerbose compile.sh \

          -e "/bazel_build /a\  --verbose_failures \\\\" \

          -e "/bazel_build /a\  --curses=no \\\\" \

          -e "/bazel_build /a\  --features=-layering_check \\\\" \

          -e "/bazel_build /a\  --experimental_strict_java_deps=off \\\\" \

          -e "/bazel_build /a\  --strict_proto_deps=off \\\\" \

          -e "/bazel_build /a\  --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \

          -e "/bazel_build /a\  --tool_java_runtime_version=local_jdk_17 \\\\" \

          -e "/bazel_build /a\  --java_runtime_version=local_jdk_17 \\\\" \

          -e "/bazel_build /a\  --tool_java_language_version=17 \\\\" \

          -e "/bazel_build /a\  --java_language_version=17 \\\\" \

          -e "/bazel_build /a\  --tool_java_runtime_version=local_jdk_21 \\\\" \

          -e "/bazel_build /a\  --java_runtime_version=local_jdk_21 \\\\" \

          -e "/bazel_build /a\  --tool_java_language_version=21 \\\\" \

          -e "/bazel_build /a\  --java_language_version=21 \\\\" \

          -e "/bazel_build /a\  --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \

          -e "/bazel_build /a\  --vendor_dir=../vendor_dir \\\\" \

          -e "/bazel_build /a\  --repository_disable_download \\\\" \

          -e "/bazel_build /a\  --announce_rc \\\\" \

          -e "/bazel_build /a\  --nobuild_python_zip \\\\" \

        # Also build parser_deploy.jar with bootstrap bazel

        # TODO: Turn into a proper patch

  # Bazel starts a local server and needs to bind a local address.

  __darwinAllowLocalNetworking = true;

  buildInputs = [ buildJdk bashWithDefaultShellUtils ] ++ defaultShellUtils;

  buildInputs = [

    buildJdk

    bashWithDefaultShellUtils

  ] ++ defaultShellUtils;

  # when a command can’t be found in a bazel build, you might also

  # need to add it to `defaultShellPath`.

  nativeBuildInputs = [

  nativeBuildInputs =

    [

      installShellFiles

      makeWrapper

      python3

      which

      zip

      python3.pkgs.absl-py # Needed to build fish completion

  ] ++ lib.optionals (stdenv.hostPlatform.isDarwin) [

    ]

    ++ lib.optionals (stdenv.hostPlatform.isDarwin) [

      cctools

      libcxx

      Foundation

    mkdir bazel_src

    shopt -s dotglob extglob

    mv !(bazel_src) bazel_src

    # Augment bundled repository_cache with our extra paths

    mkdir vendor_dir

    ${lndir}/bin/lndir ${bazelDeps}/vendor_dir vendor_dir

    rm vendor_dir/VENDOR.bazel

    find vendor_dir -maxdepth 1 -type d -printf "pin(\"@@%P\")\n" > vendor_dir/VENDOR.bazel

  '';

  buildPhase = ''

    runHook preBuild

    # Increasing memory during compilation might be necessary.

    # export BAZEL_JAVAC_OPTS="-J-Xmx2g -J-Xms200m"

    export HOME=$(mktemp -d)

    # If EMBED_LABEL isn't set, it'd be auto-detected from CHANGELOG.md

    # and `git rev-parse --short HEAD` which would result in

    # Note that .bazelversion is always correct and is based on bazel-*

    # executable name, version checks should work fine

    export EMBED_LABEL="${version}- (@non-git)"

    echo "Stage 1 - Running bazel bootstrap script"

    ${bash}/bin/bash ./bazel_src/compile.sh

    #!${runtimeShell} -e

    exit 1

    EOF

    chmod +x tools/bazel

    # first call should fail if tools/bazel is used

  # Save paths to hardcoded dependencies so Nix can detect them.

  # This is needed because the templates get tar’d up into a .jar.

  postFixup = ''

  postFixup =

    ''

      mkdir -p $out/nix-support

      echo "${defaultShellPath}" >> $out/nix-support/depends

      # The string literal specifying the path to the bazel-rc file is sometimes

      # stored non-contiguously in the binary due to gcc optimisations, which leads

      # Nix to miss the hash when scanning for dependencies

      echo "${bazelRC}" >> $out/nix-support/depends

  '' + lib.optionalString stdenv.hostPlatform.isDarwin ''

    ''

    + lib.optionalString stdenv.hostPlatform.isDarwin ''

      echo "${cctools}" >> $out/nix-support/depends

    '';

    #     nix-build . -A bazel_7.tests

    #

    # in the nixpkgs checkout root to exercise them locally.

    tests = callPackage ./tests.nix {

      inherit Foundation bazel_self lockfile repoCache;

    };

    # tests = callPackage ./tests.nix {

    #   inherit Foundation bazel_self lockfile repoCache;

    # };

    # TODO tests have not been updated yet and will likely need a rewrite

    # tests = callPackage ./tests.nix { inherit Foundation bazelDeps bazel_self; };

    # For ease of debugging

    inherit distDir repoCache lockfile;

    inherit bazelDeps bazelFhs bazelBootstrap;

  };

}

--- a/src/test/shell/bazel/list_source_repository.bzl

+++ b/src/test/shell/bazel/list_source_repository.bzl

@@ -32,7 +32,8 @@ def _impl(rctx):

     if "SRCS_EXCLUDES" in rctx.os.environ:

         srcs_excludes = rctx.os.environ["SRCS_EXCLUDES"]

     r = rctx.execute(["find", "-L", str(workspace), "-type", "f"])

-    rctx.file("find.result.raw", r.stdout.replace(str(workspace) + "/", ""))

+    stdout = "\n".join(sorted(r.stdout.splitlines()))

+    rctx.file("find.result.raw", stdout.replace(str(workspace) + "/", ""))

     rctx.file("BUILD", """

 genrule(

   name = "sources",

  bazel_7 = darwin.apple_sdk_11_0.callPackage ../development/tools/build-managers/bazel/bazel_7 {

    inherit (darwin) sigtool;

    inherit (darwin.apple_sdk_11_0.frameworks) CoreFoundation CoreServices Foundation IOKit;

    buildJdk = jdk17_headless;

    runJdk = jdk17_headless;

    buildJdk = jdk21_headless;

    runJdk = jdk21_headless;

    stdenv = if stdenv.hostPlatform.isDarwin then darwin.apple_sdk_11_0.stdenv

      else if stdenv.cc.isClang then llvmPackages.stdenv

      else stdenv;