Merge pull request #330481 from philiptaron/issue-208242/stdenv/darwin/make-bootstrap-tools.nix

{

  lib,

  stdenv,

  bootstrapTools,

  unpack,

}:

builtins.derivation {

  inherit (stdenv.hostPlatform) system;

  name = "bootstrap-tools";

  builder = "${unpack}/bin/bash";

  args = [

    "${unpack}/bootstrap-tools-unpack.sh"

    bootstrapTools

  ];

  PATH = lib.makeBinPath [

    (builtins.placeholder "out")

    unpack

  ];

  allowedReferences = [ "out" ];

}

{ pkgspath ? ../../.., test-pkgspath ? pkgspath

, localSystem ? { system = builtins.currentSystem; }

, crossSystem ? null

, bootstrapFiles ? null

{

  pkgspath ? ../../..,

  test-pkgspath ? pkgspath,

  localSystem ? {

    system = builtins.currentSystem;

  },

  crossSystem ? null,

  bootstrapFiles ? null,

}:

let cross = if crossSystem != null

      then { inherit crossSystem; }

      else {};

    custom-bootstrap = if bootstrapFiles != null

      then { stdenvStages = args:

              let args' = args // { bootstrapFiles = bootstrapFiles; };

              in (import "${pkgspath}/pkgs/stdenv/darwin" args');

           }

      else {};

in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap);

rec {

  build = stdenv.mkDerivation {

    name = "stdenv-bootstrap-tools";

    nativeBuildInputs = [ dumpnar nukeReferences ];

    buildCommand = let

      inherit (lib)

        getBin

        getDev

        getLib

        ;

      coreutils_ = (coreutils.override (args: {

        # We want coreutils without ACL support.

        aclSupport = false;

        # Cannot use a single binary build, or it gets dynamically linked against gmp.

        singleBinary = false;

      })).overrideAttrs (oa: {

        # Increase header size to be able to inject extra RPATHs. Otherwise

        # x86_64-darwin build fails as:

        #    https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv

        NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names";

      });

      # Avoid messing with libkrb5 and libnghttp2.

      curl_ = curlMinimal.override (args: {

        gssSupport = false;

        http2Support = false;

        scpSupport = false;

      });

      unpackScript = writeText "bootstrap-tools-unpack.sh" ''

        set -euo pipefail

        echo Unpacking the bootstrap tools... >&2

        mkdir $out

        tar xf "$1" -C $out

        updateInstallName() {

          local path="$1"

          cp "$path" "$path.new"

          install_name_tool -id "$path" "$path.new"

          codesign -f -i "$(basename "$path")" -s - "$path.new"

          mv -f "$path.new" "$path"

        }

        find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do

          updateInstallName "$lib"

        done

        # as is a wrapper around clang. need to replace the nuked store paths

        sed -i 's|/.*/bin/|'"$out"'/bin/|' $out/bin/as

        # Provide a gunzip script.

        cat > $out/bin/gunzip <<EOF

        #!$out/bin/sh

        exec $out/bin/gzip -d "\$@"

        EOF

        chmod +x $out/bin/gunzip

        # Provide fgrep/egrep.

        echo "#! $out/bin/sh" > $out/bin/egrep

        echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep

        echo "#! $out/bin/sh" > $out/bin/fgrep

        echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep

        cat >$out/bin/dsymutil << EOF

        #!$out/bin/sh

        EOF

        chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil

    '';

let

  cross = if crossSystem != null then { inherit crossSystem; } else { };

  custom-bootstrap =

    if bootstrapFiles != null then

      {

        stdenvStages =

          args:

          let

            args' = args // {

              bootstrapFiles = bootstrapFiles;

            };

          in

    ''

      mkdir -p $out/bin $out/lib $out/lib/darwin

      ${lib.optionalString stdenv.targetPlatform.isx86_64 ''

        # Copy libSystem's .o files for various low-level boot stuff.

        cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib

        # Resolv is actually a link to another package, so let's copy it properly

        cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib

      ''}

      cp -rL ${getDev darwin.Libsystem}/include $out

      chmod -R u+w $out/include

      cp -rL ${getDev libiconv}/include/* $out/include

      cp -rL ${getDev gnugrep.pcre2}/include/* $out/include

      mv $out/include $out/include-Libsystem

      # Copy binutils.

      for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do

        cp ${getBin darwin.binutils-unwrapped}/bin/$i $out/bin

      done

      cp -d ${getLib ld64}/lib/libcodedirectory*.dylib $out/lib

      # Copy coreutils, bash, etc.

      cp ${getBin coreutils_}/bin/* $out/bin

      (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users)

      cp -d ${getBin bash}/bin/{ba,}sh $out/bin

      cp -d ${getBin diffutils}/bin/* $out/bin

      cp ${getBin findutils}/bin/{find,xargs} $out/bin

      cp -d ${getBin gawk}/bin/{g,}awk $out/bin

      cp -d ${getBin gnugrep}/bin/grep $out/bin

      cp -d ${getBin gnumake}/bin/* $out/bin

      cp -d ${getBin gnused}/bin/* $out/bin

      cp -d ${getBin patch}/bin/* $out/bin

      cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib

      cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib

      cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib

      cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib

      cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib

      # copy package extraction tools

      cp -d ${getBin bzip2}/bin/b{,un}zip2 $out/bin

      cp ${getBin cpio}/bin/cpio $out/bin

      cp ${getBin gnutar}/bin/tar $out/bin

      cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip

      cp ${getBin pbzx}/bin/pbzx $out/bin

      cp ${getBin xz}/bin/xz $out/bin

      cp -d ${getLib bzip2}/lib/libbz2*.dylib $out/lib

      cp -d ${getLib gmpxx}/lib/libgmp*.dylib $out/lib

      cp -d ${getLib xar}/lib/libxar*.dylib $out/lib

      cp -d ${getLib xz}/lib/liblzma*.dylib $out/lib

      cp -d ${getLib zlib}/lib/libz*.dylib $out/lib

      # This used to be in-nixpkgs, but now is in the bundle

      # because I can't be bothered to make it partially static

      cp ${getBin curl_}/bin/curl $out/bin

      cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib

      cp -d ${getLib openssl}/lib/*.dylib $out/lib

      # Copy what we need of clang

      cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin

      cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib

      cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib

      cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib

      mkdir -p $out/lib/darwin

      cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a  $out/lib/darwin

      cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib

      cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib

      cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib

      mkdir $out/include

      cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include

      # copy .tbd assembly utils

      cp ${getBin darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin

      cp -d ${getLib libyaml}/lib/libyaml*.dylib $out/lib

      # copy sigtool

      cp -d ${getBin darwin.sigtool}/bin/{codesign,sigtool} $out/bin

      cp -d ${getLib darwin.libtapi}/lib/libtapi*.dylib $out/lib

      # tools needed to unpack bootstrap archive

      mkdir -p unpack/bin unpack/lib

      cp -d ${getBin bash}/bin/{bash,sh} unpack/bin

      cp ${getBin coreutils_}/bin/mkdir unpack/bin

      cp ${getBin gnutar}/bin/tar unpack/bin

      cp ${getBin xz}/bin/xz unpack/bin

      cp -d ${getLib gettext}/lib/libintl*.dylib unpack/lib

      cp -d ${getLib libiconv}/lib/lib*.dylib unpack/lib

      cp -d ${getLib xz}/lib/liblzma*.dylib unpack/lib

      cp ${unpackScript} unpack/bootstrap-tools-unpack.sh

      #

      # All files copied. Perform processing to update references to point into

      # the archive

      #

      chmod -R u+w $out unpack

      # - change nix store library paths to use @rpath/library

      # - if needed add an rpath containing lib/

      # - strip executable

      rpathify() {

        local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true)

        local lib rpath

        for lib in $libs; do

          ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1"

        done

        case "$(dirname "$1")" in

        */bin)

          # Strip executables even further

          ${stdenv.cc.targetPrefix}strip "$i"

          rpath='@executable_path/../lib'

          ;;

        */lib)

          # the '/.' suffix is required

          rpath='@loader_path/.'

          ;;

        */lib/darwin)

          rpath='@loader_path/..'

          ;;

        *)

          echo unkown executable $1 >&2

          exit 1

          ;;

        esac

        # if shared object contains references add an rpath to lib/

        if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then

          ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1"

        fi

      }

      # check that linked library paths exist in lib

      # must be run after rpathify is performed

      checkDeps() {

        local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^      ]*' || true)

        local lib

        shopt -s extglob

        for lib in $deps; do

          local root="''${1/\/@(lib|bin)\/*}"

          if [[ ! -e $root/''${lib/@rpath/lib} ]]; then

            echo "error: $1 missing lib for $lib" >&2

            exit 1

          fi

        done

        shopt -u extglob

          (import "${pkgspath}/pkgs/stdenv/darwin" args');

      }

    else

      { };

      for i in {unpack,$out}/bin/* {unpack,$out}/lib{,/darwin}/*.dylib; do

        if [[ ! -L $i ]] && isMachO "$i"; then

          rpathify "$i"

          checkDeps "$i"

        fi

      done

  pkgs = import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap);

      nuke-refs {unpack,$out}/bin/*

      nuke-refs {unpack,$out}/lib/*

      nuke-refs $out/lib/darwin/*

  build = pkgs.callPackage ./stdenv-bootstrap-tools.nix { };

      mkdir $out/.pack

      mv $out/* $out/.pack

      mv $out/.pack $out/pack

      mkdir $out/on-server

      cp -r unpack $out

      XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \

        --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack .

      dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz

    '';

    allowedReferences = [];

    meta = {

      maintainers = [ lib.maintainers.copumpkin ];

    };

  bootstrapTools = pkgs.callPackage ./bootstrap-tools.nix {

    inherit (build.bootstrapFiles) bootstrapTools unpack;

  };

  bootstrapFiles = {

    bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz";

    unpack = runCommand "unpack" { allowedReferences = []; } ''

      cp -r ${build}/unpack $out

    '';

  };

  bootstrapTools = derivation {

    inherit (stdenv.hostPlatform) system;

    name = "bootstrap-tools";

    builder = "${bootstrapFiles.unpack}/bin/bash";

    args = [

      "${bootstrapFiles.unpack}/bootstrap-tools-unpack.sh"

        bootstrapFiles.bootstrapTools

    ];

    PATH = lib.makeBinPath [

      (placeholder "out")

      bootstrapFiles.unpack

    ];

    allowedReferences = [ "out" ];

  };

  test = derivation {

    name = "test-bootstrap-tools";

    inherit (stdenv.hostPlatform) system;

    builder = "${bootstrapTools}/bin/bash";

    args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ];

    PATH = lib.makeBinPath [ bootstrapTools ];

    tools = bootstrapTools;

    "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion;

    # Create a pure environment where we use just what's in the bootstrap tools.

    buildCommand = ''

      mkdir -p $out/bin

      for exe in $tools/bin/*; do

        [[ $exe =~ bunzip2|codesign.*|false|install_name_tool|ld|lipo|pbzx|ranlib|rewrite-tbd|sigtool ]] && continue

        $exe --version > /dev/null || { echo $exe failed >&2; exit 1; }

      done

      # run all exes that don't take a --version flag

      bunzip2 -h

      codesign --help

      codesign_allocate -i $tools/bin/true -r -o true

      false || (($? == 1))

      install_name_tool -id true true

      ld -v

      lipo -info true

      pbzx -v

      # ranlib gets tested bulding hello

      rewrite-tbd </dev/null

      sigtool -h

      rm true

      # The grep will return a nonzero exit code if there is no match, and we want to assert that we have

      # an SSL-capable curl

      curl --version | grep SSL

      # This approximates a bootstrap version of libSystem can that be

      # assembled via fetchurl. Adapted from main libSystem expression.

      mkdir libSystem-boot

      cp -vr \

        ${stdenv.cc.libc_dev}/lib/libSystem.B.tbd \

        ${stdenv.cc.libc_dev}/lib/system \

        libSystem-boot

      sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd

      ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd

      # End of bootstrap libSystem

      export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot"

      export CPP="clang -E $flags"

      export CC="clang $flags"

      export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1"

      # NOTE: These tests do a separate 'install' step (using cp), because

      # having clang write directly to the final location apparently will make

      # running the executable fail signature verification. (SIGKILL'd)

      #

      # Suspect this is creating a corrupt entry in the kernel cache, but it is

      # unique to cctools ld. (The problem goes away with `-fuse-ld=lld`.)

      echo '#include <stdio.h>' >> hello1.c

      echo '#include <float.h>' >> hello1.c

      echo '#include <limits.h>' >> hello1.c

      echo 'int main() { printf("Hello World\n"); return 0; }' >> hello1.c

      $CC -o hello1 hello1.c

      cp hello1 $out/bin/

      $out/bin/hello1

      echo '#include <iostream>' >> hello3.cc

      echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc

      $CXX -v -o hello3 hello3.cc

      cp hello3 $out/bin/

      $out/bin/hello3

      # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked.

      # using -Wl,-flat_namespace is required to generate an error

      mkdir libtest/

      ln -s $tools/lib/libc++.dylib libtest/

      clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \

        --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc

      tar xvf ${hello.src}

      cd hello-*

      # hello configure detects -liconv is needed but doesn't add to the link step

      LDFLAGS=-liconv ./configure --prefix=$out

      make

      make install

      $out/bin/hello

    '';

  };

  test = pkgs.callPackage ./test-bootstrap-tools.nix { inherit bootstrapTools; };

  # The ultimate test: bootstrap a whole stdenv from the tools specified above and get a package set out of it

  # eg: nix-build -A freshBootstrapTools.test-pkgs.stdenv

    # that platform.

    localSystem = if crossSystem != null then crossSystem else localSystem;

    stdenvStages = args: let

        args' = args // { inherit bootstrapFiles; };

      in (import (test-pkgspath + "/pkgs/stdenv/darwin") args');

    stdenvStages =

      args:

      let

        args' = args // {

          inherit (build) bootstrapFiles;

        };

      in

      (import (test-pkgspath + "/pkgs/stdenv/darwin") args');

  };

in

{

  inherit

    build

    bootstrapTools

    test

    test-pkgs

    ;

  inherit (build) bootstrapFiles;

}

{

  lib,

  stdenv,

  bootstrapTools,

  hello,

}:

builtins.derivation {

  name = "test-bootstrap-tools";

  inherit (stdenv.hostPlatform) system;

  builder = "${bootstrapTools}/bin/bash";

  args = [

    "-euo"

    "pipefail"

    "-c"

    "eval \"$buildCommand\""

  ];

  PATH = lib.makeBinPath [ bootstrapTools ];

  tools = bootstrapTools;

  "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion;

  # Create a pure environment where we use just what's in the bootstrap tools.

  buildCommand = ''

    mkdir -p $out/bin

    for exe in $tools/bin/*; do

      [[ $exe =~ bunzip2|codesign.*|false|install_name_tool|ld|lipo|pbzx|ranlib|rewrite-tbd|sigtool ]] && continue

      $exe --version > /dev/null || { echo $exe failed >&2; exit 1; }

    done

    # run all exes that don't take a --version flag

    bunzip2 -h

    codesign --help

    codesign_allocate -i $tools/bin/true -r -o true

    false || (($? == 1))

    install_name_tool -id true true

    ld -v

    lipo -info true

    pbzx -v

    # ranlib gets tested bulding hello

    rewrite-tbd </dev/null

    sigtool -h

    rm true

    # The grep will return a nonzero exit code if there is no match, and we want to assert that we have

    # an SSL-capable curl

    curl --version | grep SSL

    # This approximates a bootstrap version of libSystem can that be

    # assembled via fetchurl. Adapted from main libSystem expression.

    mkdir libSystem-boot

    cp -vr \

      ${stdenv.cc.libc_dev}/lib/libSystem.B.tbd \

      ${stdenv.cc.libc_dev}/lib/system \

      libSystem-boot

    sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd

    ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd

    # End of bootstrap libSystem

    export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot"

    export CPP="clang -E $flags"

    export CC="clang $flags"

    export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1"

    # NOTE: These tests do a separate 'install' step (using cp), because

    # having clang write directly to the final location apparently will make

    # running the executable fail signature verification. (SIGKILL'd)

    #

    # Suspect this is creating a corrupt entry in the kernel cache, but it is

    # unique to cctools ld. (The problem goes away with `-fuse-ld=lld`.)

    echo '#include <stdio.h>' >> hello1.c

    echo '#include <float.h>' >> hello1.c

    echo '#include <limits.h>' >> hello1.c

    echo 'int main() { printf("Hello World\n"); return 0; }' >> hello1.c

    $CC -o hello1 hello1.c

    cp hello1 $out/bin/

    $out/bin/hello1

    echo '#include <iostream>' >> hello3.cc

    echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc

    $CXX -v -o hello3 hello3.cc

    cp hello3 $out/bin/

    $out/bin/hello3

    # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked.

    # using -Wl,-flat_namespace is required to generate an error

    mkdir libtest/

    ln -s $tools/lib/libc++.dylib libtest/

    clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \

      --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc

    tar xvf ${hello.src}

    cd hello-*

    # hello configure detects -liconv is needed but doesn't add to the link step

    LDFLAGS=-liconv ./configure --prefix=$out

    make

    make install

    $out/bin/hello

  '';

}