Unverified Commit b3108303 authored by annalee's avatar annalee
Browse files

stdenv.darwin.make-boostrap-tools: update tools and unpin LLVM11 

- unpin LLVM11. fix discrepancy with freshBootstrapTools and the tools
  built on hydra. pinning the stdenv for the hydra build doesn't pin the
  tools as the included packages are able to change.

- remove unused LLVM tools & libs which reduces the uncompressed and
  compressed file sizes by more than 1/2. compressed tarball is now 40M
  and uncompressed is around 200M

- add @loader_path/. to dylibs that reference other libs in the archive.
  this is needed for libraries with re-exports.

- validate shared objects with @rpath references contain the reference
  in lib

- add a test to verify that the @loader_path/ works for libc++ as it
  re-exports libc++abi
parent dcc63e54

pkgs/stdenv/darwin/make-bootstrap-tools.nix

+145 −104
Original line number Diff line number Diff line
@@ -16,6 +16,18 @@ let cross = if crossSystem != null 
in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap);



rec {

  build = stdenv.mkDerivation {

    name = "stdenv-bootstrap-tools";



    nativeBuildInputs = [ dumpnar nukeReferences ];



    buildCommand = let

      inherit (lib)

        getBin

        getDev

        getLib

        ;



      coreutils_ = (coreutils.override (args: {

        # We want coreutils without ACL support.

        aclSupport = false;
@@ -30,23 +42,20 @@ rec { 


      cctools_ = darwin.cctools;



  # Avoid debugging larger changes for now.

  bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; });



      # Avoid messing with libkrb5 and libnghttp2.

  curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; });



  build = stdenv.mkDerivation {

    name = "stdenv-bootstrap-tools";

      curl_ = curlMinimal.override (args: {

        gssSupport = false;

        http2Support = false;

        scpSupport = false;

      });



    nativeBuildInputs = [ nukeReferences dumpnar ];

      gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: {

        configureFlags = [

          "--disable-nls"

        ] ++ old.configureFlags or [];

      });



    buildCommand = let

      inherit (lib)

        getBin

        getDev

        getLib

        ;

      xz_ = xz.override { enableStatic = true; };

    in

    ''

      mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin
@@ -61,7 +70,6 @@ rec { 


      cp -rL ${getDev darwin.Libsystem}/include $out

      chmod -R u+w $out/include

      cp -rL ${getDev darwin.ICU}/include* $out/include

      cp -rL ${getDev libiconv}/include/* $out/include

      cp -rL ${getDev gnugrep.pcre2}/include/* $out/include

      mv $out/include $out/include-Libsystem
@@ -81,7 +89,7 @@ rec { 
      cp -d ${getBin gawk}/bin/awk $out/bin

      cp ${getBin gnutar}/bin/tar $out/bin

      cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip

      cp ${getBin bzip2_}/bin/bzip2 $out/bin

      cp ${getBin bzip2}/bin/bzip2 $out/bin

      ln -s bzip2 $out/bin/bunzip2

      cp -d ${getBin gnumake}/bin/* $out/bin

      cp -d ${getBin patch}/bin/* $out/bin
@@ -92,7 +100,6 @@ rec { 
      # because I can't be bothered to make it partially static

      cp ${getBin curl_}/bin/curl $out/bin

      cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib

      cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib

      cp -d ${getLib openssl}/lib/*.dylib $out/lib



      cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib
@@ -103,17 +110,14 @@ rec { 
      cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib



      # Copy what we need of clang

      cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin

      cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib

      cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin

      cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib

      cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib



      cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib

    ''

    # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11

    + lib.optionalString (llvmPackages ? libcxxabi) ''

      cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib

    '' + ''

      cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin

      cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib

      mkdir -p $out/lib/darwin

      cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a  $out/lib/darwin

      cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib

      cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib

      cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib
@@ -133,7 +137,6 @@ rec { 
      cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin

      cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin



      cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib

      cp -d ${getLib zlib}/lib/libz.*       $out/lib

      cp -d ${getLib gmpxx}/lib/libgmp*.*   $out/lib

      cp -d ${getLib xz}/lib/liblzma*.*     $out/lib
@@ -145,63 +148,103 @@ rec { 


      cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib



      cp -rd ${getLib pkgs.darwin.CF}/Library $out

      ${lib.optionalString stdenv.targetPlatform.isAarch64 ''

        cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/

      ''}

      # tools needed to unpack bootstrap archive. they should not contain any

      # external references. we will process them like the other tools but

      # perform some additional checks and will not pack them into the archive.

      mkdir -p unpack/bin

      cp ${getBin bash}/bin/bash unpack/bin

      ln -s bash unpack/bin/sh

      cp ${getBin coreutils_}/bin/mkdir unpack/bin

      cp ${getBin gnutar_}/bin/tar unpack/bin

      cp ${getBin xz_}/bin/xz unpack/bin



      chmod -R u+w $out

      #

      # All files copied. Perform processing to update references to point into

      # the archive

      #



      nuke-refs $out/bin/*

      chmod -R u+w $out unpack



      # - change nix store library paths to use @rpath/library

      # - if needed add an rpath containing lib/

      # - strip executable

      rpathify() {

        local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true

        local newlib

        local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true)

        local lib rpath

        for lib in $libs; do

          ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1"

        done

      }



        case "$(dirname "$1")" in

        */bin)

          # Strip executables even further

      for i in $out/bin/*; do

        if [[ ! -L $i ]] && isMachO "$i"; then

          chmod +w $i

          ${stdenv.cc.targetPrefix}strip $i || true

          ${stdenv.cc.targetPrefix}strip "$i"

          rpath='@executable_path/../lib'

          ;;

        */lib)

          # the '/.' suffix is required

          rpath='@loader_path/.'

          ;;

        */lib/darwin)

          rpath='@loader_path/..'

          ;;

        *)

          echo unkown executable $1 >&2

          exit 1

          ;;

        esac



        # if shared object contains references add an rpath to lib/

        if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then

          ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1"

        fi

      done

      }



      for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do

        if [[ ! -L "$i" ]]; then

          rpathify $i

      # check that linked library paths exist in $out/lib

      # must be run after rpathify is performed

      checkDeps() {

        local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^      ]*' || true)

        local lib

        for lib in $deps; do

          if [[ ! -e $out/''${lib/@rpath/lib} ]]; then

            echo "error: $1 missing lib for $lib" >&2

            exit 1

          fi

        done

      }



      for i in $out/bin/*; do

        if [[ ! -L "$i" ]] && isMachO "$i"; then

          ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i

      for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do

        if [[ ! -L $i ]] && isMachO "$i"; then

          rpathify "$i"

          checkDeps "$i"

        fi

      done



      ${if stdenv.targetPlatform.isx86_64 then ''

        rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation

      '' else ''

        sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \

          $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd

      ''}



      nuke-refs $out/bin/*

      nuke-refs $out/lib/*

      nuke-refs $out/lib/system/*

      nuke-refs $out/lib/darwin/*

      ${lib.optionalString stdenv.targetPlatform.isx86_64 ''

        nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation

      ''}

      nuke-refs $out/lib/system/*

      nuke-refs unpack/bin/*



      mkdir $out/.pack

      mv $out/* $out/.pack

      mv $out/.pack $out/pack



      # validate that tools contain no references into the archive

      for tool in unpack/bin/*; do

        deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true)

        if [[ -n "$deps" ]]; then

          printf "error: $tool is not self contained\n$deps\n" >&2

          exit 1

        fi

      done



      mkdir $out/on-server

      dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz

      cp -r unpack $out



      XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \

        --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack .

      dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz

    '';



    allowedReferences = [];
@@ -211,44 +254,42 @@ rec { 
    };

  };



  dist = stdenv.mkDerivation {

    name = "stdenv-bootstrap-tools";



    buildCommand = ''

  dist = runCommand "stdenv-bootstrap-tools" {} ''

    mkdir -p $out/nix-support

      echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products

    echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products

    echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products

  '';

  };



  bootstrapFiles = {

    tools = "${build}/pack";

    bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz";

    unpack = runCommand "unpack" { allowedReferences = []; } ''

      cp -r ${build}/unpack $out

    '';

  };



  bootstrapTools = derivation {

    inherit (stdenv.hostPlatform) system;



    name = "bootstrap-tools";

    builder = "${bootstrapFiles.tools}/bin/bash";



    # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly,

    # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the

    # patch-bootstrap-tools.sh for the next round of bootstrap tools.

    args = [ ./patch-bootstrap-tools-next.sh ];

    builder = "${bootstrapFiles.unpack}/bin/bash";



    inherit (bootstrapFiles) tools;

    args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ];

    PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ];



    allowedReferences = [ "out" ];

  };



  test = stdenv.mkDerivation {

    name = "test";



    realBuilder = "${bootstrapTools}/bin/bash";



  test = derivation {

    name = "test-bootstrap-tools";

    inherit (stdenv.hostPlatform) system;

    builder = "${bootstrapTools}/bin/bash";

    args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ];

    PATH = lib.makeBinPath [ bootstrapTools ];

    tools = bootstrapTools;

    buildCommand = ''

    "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion;



    # Create a pure environment where we use just what's in the bootstrap tools.

      export PATH=$tools/bin

    buildCommand = ''



      ls -l

      mkdir $out
@@ -275,16 +316,15 @@ rec { 
        ${stdenv.cc.libc_dev}/lib/system \

        libSystem-boot



      substituteInPlace libSystem-boot/libSystem.B.tbd \

        --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/"

      sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd

      ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd

      # End of bootstrap libSystem



      export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot"



      export CPP="clang -E $flags"

      export CC="clang $flags -rpath $tools/lib"

      export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib"

      export CC="clang $flags"

      export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1"



      # NOTE: These tests do a separate 'install' step (using cp), because

      # having clang write directly to the final location apparently will make
@@ -301,22 +341,23 @@ rec { 
      cp hello1 $out/bin/

      $out/bin/hello1



      echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c

      echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c

      $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c

      cp hello2 $out/bin/

      $out/bin/hello2



      echo '#include <iostream>' >> hello3.cc

      echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc

      $CXX -v -o hello3 hello3.cc

      cp hello3 $out/bin/

      $out/bin/hello3



      # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked.

      # using -Wl,-flat_namespace is required to generate an error

      mkdir libtest/

      ln -s $tools/lib/libc++.dylib libtest/

      clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \

        --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc



      tar xvf ${hello.src}

      cd hello-*

      # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331

      am_cv_func_iconv=no ./configure --prefix=$out

      # hello configure detects -liconv is needed but doesn't add to the link step

      LDFLAGS=-liconv ./configure --prefix=$out

      make

      make install

      $out/bin/hello

pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh→pkgs/stdenv/darwin/patch-bootstrap-tools.sh

+3 −4
Original line number Diff line number Diff line 
set -euo pipefail



export PATH=$tools/bin



cp -R $tools $out

chmod -R u+w $out

echo Unpacking the bootstrap tools... >&2

mkdir $out

tar xf "$1" -C $out



updateInstallName() {

  local path="$1"