Loading maintainers/scripts/bootstrap-files/refresh-tarballs.bash +30 −14 Original line number Diff line number Diff line Loading @@ -67,14 +67,8 @@ NATIVE_TARGETS=( i686-unknown-linux-gnu x86_64-unknown-linux-gnu x86_64-unknown-linux-musl # TODO: add darwin here once a few prerequisites are satisfied: # - bootstrap-files are factored out into a separate file # - the build artifacts are factored out into an `on-server` # directory. Right onw if does not match `linux` layout. # #aarch64-apple-darwin #x86_64-apple-darwin aarch64-apple-darwin x86_64-apple-darwin ) is_native() { Loading Loading @@ -106,6 +100,18 @@ is_cross() { return 1 } nar_sri_get() { local ouput sri output=$(nix-build --expr \ 'import <nix/fetchurl.nix> { url = "'"$1"'"; unpack = true; }' 2>&1 || true) sri=$(echo "$output" | awk '/^\s+got:\s+/{ print $2 }') [[ -z "$sri" ]] && die "$output" echo "$sri" } # collect passed options targets=() Loading Loading @@ -222,6 +228,7 @@ EOF case "$fname" in bootstrap-tools.tar.xz) attr=bootstrapTools ;; busybox) attr=$fname ;; unpack.nar.xz) attr=unpack ;; *) die "Don't know how to map '$fname' to attribute name. Please update me." esac Loading @@ -231,16 +238,25 @@ EOF executable_arg="--executable" executable_nix="executable = true;" fi unpack_nix= if [[ $fname = *.nar.* ]]; then unpack_nix="unpack = true;" sri=$(nar_sri_get "file://$p") else sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p") [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'" sri=$(nix-hash --to-sri "sha256:$sha256") [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form" fi # individual file entries cat <<EOF $attr = import <nix/fetchurl.nix> { url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname"; hash = "${sri}";$(printf "\n%s" "${executable_nix}") hash = "${sri}";$( [[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}" [[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}" ) }; EOF done Loading pkgs/stdenv/darwin/make-bootstrap-tools.nix +246 −149 Original line number Diff line number Diff line Loading @@ -16,6 +16,18 @@ let cross = if crossSystem != null in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); rec { build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; nativeBuildInputs = [ dumpnar nukeReferences ]; buildCommand = let inherit (lib) getBin getDev getLib ; coreutils_ = (coreutils.override (args: { # We want coreutils without ACL support. aclSupport = false; Loading @@ -30,171 +42,251 @@ rec { cctools_ = darwin.cctools; # Avoid debugging larger changes for now. bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; }); # Avoid messing with libkrb5 and libnghttp2. curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; }); curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; scpSupport = false; }); build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: { configureFlags = [ "--disable-nls" ] ++ old.configureFlags or []; }); nativeBuildInputs = [ nukeReferences dumpnar ]; xz_ = xz.override { enableStatic = true; }; buildCommand = '' unpackScript = writeText "bootstrap-tools-unpack.sh" '' set -euo pipefail echo Unpacking the bootstrap tools... >&2 mkdir $out tar xf "$1" -C $out updateInstallName() { local path="$1" cp "$path" "$path.new" install_name_tool -id "$path" "$path.new" codesign -f -i "$(basename "$path")" -s - "$path.new" mv -f "$path.new" "$path" } find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do updateInstallName "$lib" done # Provide a gunzip script. cat > $out/bin/gunzip <<EOF #!$out/bin/sh exec $out/bin/gzip -d "\$@" EOF chmod +x $out/bin/gunzip # Provide fgrep/egrep. echo "#! $out/bin/sh" > $out/bin/egrep echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep echo "#! $out/bin/sh" > $out/bin/fgrep echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep cat >$out/bin/dsymutil << EOF #!$out/bin/sh EOF chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil ''; in '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin ${lib.optionalString stdenv.targetPlatform.isx86_64 '' # Copy libSystem's .o files for various low-level boot stuff. cp -d ${lib.getLib darwin.Libsystem}/lib/*.o $out/lib cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib # Resolv is actually a link to another package, so let's copy it properly cp -L ${lib.getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib ''} cp -rL ${darwin.Libsystem}/include $out cp -rL ${getDev darwin.Libsystem}/include $out chmod -R u+w $out/include cp -rL ${darwin.ICU}/include* $out/include cp -rL ${libiconv}/include/* $out/include cp -rL ${lib.getDev gnugrep.pcre2}/include/* $out/include cp -rL ${getDev libiconv}/include/* $out/include cp -rL ${getDev gnugrep.pcre2}/include/* $out/include mv $out/include $out/include-Libsystem # Copy coreutils, bash, etc. cp ${coreutils_}/bin/* $out/bin cp ${getBin coreutils_}/bin/* $out/bin (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) cp ${bash}/bin/bash $out/bin cp ${getBin bash}/bin/bash $out/bin ln -s bash $out/bin/sh cp ${findutils}/bin/find $out/bin cp ${findutils}/bin/xargs $out/bin cp -d ${diffutils}/bin/* $out/bin cp -d ${gnused}/bin/* $out/bin cp -d ${gnugrep}/bin/grep $out/bin cp ${gawk}/bin/gawk $out/bin cp -d ${gawk}/bin/awk $out/bin cp ${gnutar}/bin/tar $out/bin cp ${gzip}/bin/.gzip-wrapped $out/bin/gzip cp ${bzip2_.bin}/bin/bzip2 $out/bin cp ${getBin findutils}/bin/find $out/bin cp ${getBin findutils}/bin/xargs $out/bin cp -d ${getBin diffutils}/bin/* $out/bin cp -d ${getBin gnused}/bin/* $out/bin cp -d ${getBin gnugrep}/bin/grep $out/bin cp ${getBin gawk}/bin/gawk $out/bin cp -d ${getBin gawk}/bin/awk $out/bin cp ${getBin gnutar}/bin/tar $out/bin cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip cp ${getBin bzip2}/bin/bzip2 $out/bin ln -s bzip2 $out/bin/bunzip2 cp -d ${gnumake}/bin/* $out/bin cp -d ${patch}/bin/* $out/bin cp -d ${xz.bin}/bin/xz $out/bin cp ${cpio}/bin/cpio $out/bin cp -d ${getBin gnumake}/bin/* $out/bin cp -d ${getBin patch}/bin/* $out/bin cp -d ${getBin xz}/bin/xz $out/bin cp ${getBin cpio}/bin/cpio $out/bin # This used to be in-nixpkgs, but now is in the bundle # because I can't be bothered to make it partially static cp ${curl_.bin}/bin/curl $out/bin cp -d ${curl_.out}/lib/libcurl*.dylib $out/lib cp -d ${libssh2.out}/lib/libssh*.dylib $out/lib cp -d ${lib.getLib openssl}/lib/*.dylib $out/lib cp -d ${gnugrep.pcre2.out}/lib/libpcre2*.dylib $out/lib cp -d ${lib.getLib libiconv}/lib/lib*.dylib $out/lib cp -d ${lib.getLib gettext}/lib/libintl*.dylib $out/lib cp ${getBin curl_}/bin/curl $out/bin cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib cp -d ${getLib openssl}/lib/*.dylib $out/lib cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib chmod +x $out/lib/libintl*.dylib cp -d ${ncurses.out}/lib/libncurses*.dylib $out/lib cp -d ${libxml2.out}/lib/libxml2*.dylib $out/lib cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib # Copy what we need of clang cp -d ${llvmPackages.clang-unwrapped}/bin/clang* $out/bin cp -rd ${lib.getLib llvmPackages.clang-unwrapped}/lib/* $out/lib cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib cp -d ${lib.getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib '' # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11 + lib.optionalString (llvmPackages ? libcxxabi) '' cp -d ${lib.getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib '' + '' cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib cp -d ${lib.getLib llvmPackages.llvm.lib}/lib/libLLVM.dylib $out/lib cp -d ${lib.getLib libffi}/lib/libffi*.dylib $out/lib cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib mkdir -p $out/lib/darwin cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib mkdir $out/include cp -rd ${llvmPackages.libcxx.dev}/include/c++ $out/include cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include # copy .tbd assembly utils cp -d ${pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin cp -d ${lib.getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib cp -d ${getBin pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin cp -d ${getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib # copy package extraction tools cp -d ${pkgs.pbzx}/bin/pbzx $out/bin cp -d ${lib.getLib pkgs.xar}/lib/libxar*.dylib $out/lib cp -d ${pkgs.bzip2.out}/lib/libbz2*.dylib $out/lib cp -d ${getBin pkgs.pbzx}/bin/pbzx $out/bin cp -d ${getLib pkgs.xar}/lib/libxar*.dylib $out/lib cp -d ${getLib pkgs.bzip2}/lib/libbz2*.dylib $out/lib # copy sigtool cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin cp -d ${lib.getLib darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${zlib.out}/lib/libz.* $out/lib cp -d ${gmpxx.out}/lib/libgmp*.* $out/lib cp -d ${xz.out}/lib/liblzma*.* $out/lib cp -d ${getLib zlib}/lib/libz.* $out/lib cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib cp -d ${getLib xz}/lib/liblzma*.* $out/lib # Copy binutils. for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do cp ${cctools_}/bin/$i $out/bin cp ${getBin cctools_}/bin/$i $out/bin done cp -d ${lib.getLib darwin.libtapi}/lib/libtapi* $out/lib cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib cp -rd ${pkgs.darwin.CF}/Library $out ${lib.optionalString stdenv.targetPlatform.isAarch64 '' cp -rd ${pkgs.darwin.libobjc}/lib/* $out/lib/ ''} # tools needed to unpack bootstrap archive. they should not contain any # external references. we will process them like the other tools but # perform some additional checks and will not pack them into the archive. mkdir -p unpack/bin cp ${getBin bash}/bin/bash unpack/bin ln -s bash unpack/bin/sh cp ${getBin coreutils_}/bin/mkdir unpack/bin cp ${getBin gnutar_}/bin/tar unpack/bin cp ${getBin xz_}/bin/xz unpack/bin cp ${unpackScript} unpack/bootstrap-tools-unpack.sh chmod -R u+w $out # # All files copied. Perform processing to update references to point into # the archive # nuke-refs $out/bin/* chmod -R u+w $out unpack # - change nix store library paths to use @rpath/library # - if needed add an rpath containing lib/ # - strip executable rpathify() { local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true local newlib local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) local lib rpath for lib in $libs; do ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" done } case "$(dirname "$1")" in */bin) # Strip executables even further for i in $out/bin/*; do if [[ ! -L $i ]] && isMachO "$i"; then chmod +w $i ${stdenv.cc.targetPrefix}strip $i || true ${stdenv.cc.targetPrefix}strip "$i" rpath='@executable_path/../lib' ;; */lib) # the '/.' suffix is required rpath='@loader_path/.' ;; */lib/darwin) rpath='@loader_path/..' ;; *) echo unkown executable $1 >&2 exit 1 ;; esac # if shared object contains references add an rpath to lib/ if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" fi done } for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do if [[ ! -L "$i" ]]; then rpathify $i # check that linked library paths exist in $out/lib # must be run after rpathify is performed checkDeps() { local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) local lib for lib in $deps; do if [[ ! -e $out/''${lib/@rpath/lib} ]]; then echo "error: $1 missing lib for $lib" >&2 exit 1 fi done } for i in $out/bin/*; do if [[ ! -L "$i" ]] && isMachO "$i"; then ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do if [[ ! -L $i ]] && isMachO "$i"; then rpathify "$i" checkDeps "$i" fi done ${if stdenv.targetPlatform.isx86_64 then '' rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation '' else '' sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \ $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd ''} nuke-refs $out/bin/* nuke-refs $out/lib/* nuke-refs $out/lib/system/* nuke-refs $out/lib/darwin/* ${lib.optionalString stdenv.targetPlatform.isx86_64 '' nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ''} nuke-refs $out/lib/system/* nuke-refs unpack/bin/* mkdir $out/.pack mv $out/* $out/.pack mv $out/.pack $out/pack # validate that tools contain no references into the archive for tool in unpack/bin/*; do deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true) if [[ -n "$deps" ]]; then printf "error: $tool is not self contained\n$deps\n" >&2 exit 1 fi done mkdir $out/on-server dumpnar $out/pack | ${xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz cp -r unpack $out XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz ''; allowedReferences = []; Loading @@ -204,44 +296,49 @@ rec { }; }; dist = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; buildCommand = '' dist = runCommand "stdenv-bootstrap-tools" {} '' mkdir -p $out/nix-support echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products ''; }; bootstrapFiles = { tools = "${build}/pack"; bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; unpack = runCommand "unpack" { allowedReferences = []; } '' cp -r ${build}/unpack $out ''; }; bootstrapTools = derivation { inherit (stdenv.hostPlatform) system; name = "bootstrap-tools"; builder = "${bootstrapFiles.tools}/bin/bash"; builder = "${bootstrapFiles.unpack}/bin/bash"; # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly, # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the # patch-bootstrap-tools.sh for the next round of bootstrap tools. args = [ ./patch-bootstrap-tools-next.sh ]; args = [ "${bootstrapFiles.unpack}/bootstrap-tools-unpack.sh" bootstrapFiles.bootstrapTools ]; inherit (bootstrapFiles) tools; PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ]; allowedReferences = [ "out" ]; }; test = stdenv.mkDerivation { name = "test"; realBuilder = "${bootstrapTools}/bin/bash"; test = derivation { name = "test-bootstrap-tools"; inherit (stdenv.hostPlatform) system; builder = "${bootstrapTools}/bin/bash"; args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ]; PATH = lib.makeBinPath [ bootstrapTools ]; tools = bootstrapTools; buildCommand = '' "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion; # Create a pure environment where we use just what's in the bootstrap tools. export PATH=$tools/bin buildCommand = '' ls -l mkdir $out Loading @@ -268,16 +365,15 @@ rec { ${stdenv.cc.libc_dev}/lib/system \ libSystem-boot substituteInPlace libSystem-boot/libSystem.B.tbd \ --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/" sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd # End of bootstrap libSystem export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" export CC="clang $flags -rpath $tools/lib" export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib" export CC="clang $flags" export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1" # NOTE: These tests do a separate 'install' step (using cp), because # having clang write directly to the final location apparently will make Loading @@ -294,22 +390,23 @@ rec { cp hello1 $out/bin/ $out/bin/hello1 echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c cp hello2 $out/bin/ $out/bin/hello2 echo '#include <iostream>' >> hello3.cc echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc $CXX -v -o hello3 hello3.cc cp hello3 $out/bin/ $out/bin/hello3 # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked. # using -Wl,-flat_namespace is required to generate an error mkdir libtest/ ln -s $tools/lib/libc++.dylib libtest/ clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \ --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc tar xvf ${hello.src} cd hello-* # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331 am_cv_func_iconv=no ./configure --prefix=$out # hello configure detects -liconv is needed but doesn't add to the link step LDFLAGS=-liconv ./configure --prefix=$out make make install $out/bin/hello Loading pkgs/stdenv/darwin/patch-bootstrap-tools-next.shdeleted 100644 → 0 +0 −38 Original line number Diff line number Diff line set -euo pipefail export PATH=$tools/bin cp -R $tools $out chmod -R u+w $out updateInstallName() { local path="$1" cp "$path" "$path.new" install_name_tool -id "$path" "$path.new" codesign -f -i "$(basename "$path")" -s - "$path.new" mv -f "$path.new" "$path" } find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do updateInstallName "$lib" done # Provide a gunzip script. cat > $out/bin/gunzip <<EOF #!$out/bin/sh exec $out/bin/gzip -d "\$@" EOF chmod +x $out/bin/gunzip # Provide fgrep/egrep. echo "#! $out/bin/sh" > $out/bin/egrep echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep echo "#! $out/bin/sh" > $out/bin/fgrep echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep cat >$out/bin/dsymutil << EOF #!$out/bin/sh EOF chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil Loading
maintainers/scripts/bootstrap-files/refresh-tarballs.bash +30 −14 Original line number Diff line number Diff line Loading @@ -67,14 +67,8 @@ NATIVE_TARGETS=( i686-unknown-linux-gnu x86_64-unknown-linux-gnu x86_64-unknown-linux-musl # TODO: add darwin here once a few prerequisites are satisfied: # - bootstrap-files are factored out into a separate file # - the build artifacts are factored out into an `on-server` # directory. Right onw if does not match `linux` layout. # #aarch64-apple-darwin #x86_64-apple-darwin aarch64-apple-darwin x86_64-apple-darwin ) is_native() { Loading Loading @@ -106,6 +100,18 @@ is_cross() { return 1 } nar_sri_get() { local ouput sri output=$(nix-build --expr \ 'import <nix/fetchurl.nix> { url = "'"$1"'"; unpack = true; }' 2>&1 || true) sri=$(echo "$output" | awk '/^\s+got:\s+/{ print $2 }') [[ -z "$sri" ]] && die "$output" echo "$sri" } # collect passed options targets=() Loading Loading @@ -222,6 +228,7 @@ EOF case "$fname" in bootstrap-tools.tar.xz) attr=bootstrapTools ;; busybox) attr=$fname ;; unpack.nar.xz) attr=unpack ;; *) die "Don't know how to map '$fname' to attribute name. Please update me." esac Loading @@ -231,16 +238,25 @@ EOF executable_arg="--executable" executable_nix="executable = true;" fi unpack_nix= if [[ $fname = *.nar.* ]]; then unpack_nix="unpack = true;" sri=$(nar_sri_get "file://$p") else sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p") [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'" sri=$(nix-hash --to-sri "sha256:$sha256") [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form" fi # individual file entries cat <<EOF $attr = import <nix/fetchurl.nix> { url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname"; hash = "${sri}";$(printf "\n%s" "${executable_nix}") hash = "${sri}";$( [[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}" [[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}" ) }; EOF done Loading
pkgs/stdenv/darwin/make-bootstrap-tools.nix +246 −149 Original line number Diff line number Diff line Loading @@ -16,6 +16,18 @@ let cross = if crossSystem != null in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); rec { build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; nativeBuildInputs = [ dumpnar nukeReferences ]; buildCommand = let inherit (lib) getBin getDev getLib ; coreutils_ = (coreutils.override (args: { # We want coreutils without ACL support. aclSupport = false; Loading @@ -30,171 +42,251 @@ rec { cctools_ = darwin.cctools; # Avoid debugging larger changes for now. bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; }); # Avoid messing with libkrb5 and libnghttp2. curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; }); curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; scpSupport = false; }); build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: { configureFlags = [ "--disable-nls" ] ++ old.configureFlags or []; }); nativeBuildInputs = [ nukeReferences dumpnar ]; xz_ = xz.override { enableStatic = true; }; buildCommand = '' unpackScript = writeText "bootstrap-tools-unpack.sh" '' set -euo pipefail echo Unpacking the bootstrap tools... >&2 mkdir $out tar xf "$1" -C $out updateInstallName() { local path="$1" cp "$path" "$path.new" install_name_tool -id "$path" "$path.new" codesign -f -i "$(basename "$path")" -s - "$path.new" mv -f "$path.new" "$path" } find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do updateInstallName "$lib" done # Provide a gunzip script. cat > $out/bin/gunzip <<EOF #!$out/bin/sh exec $out/bin/gzip -d "\$@" EOF chmod +x $out/bin/gunzip # Provide fgrep/egrep. echo "#! $out/bin/sh" > $out/bin/egrep echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep echo "#! $out/bin/sh" > $out/bin/fgrep echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep cat >$out/bin/dsymutil << EOF #!$out/bin/sh EOF chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil ''; in '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin ${lib.optionalString stdenv.targetPlatform.isx86_64 '' # Copy libSystem's .o files for various low-level boot stuff. cp -d ${lib.getLib darwin.Libsystem}/lib/*.o $out/lib cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib # Resolv is actually a link to another package, so let's copy it properly cp -L ${lib.getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib ''} cp -rL ${darwin.Libsystem}/include $out cp -rL ${getDev darwin.Libsystem}/include $out chmod -R u+w $out/include cp -rL ${darwin.ICU}/include* $out/include cp -rL ${libiconv}/include/* $out/include cp -rL ${lib.getDev gnugrep.pcre2}/include/* $out/include cp -rL ${getDev libiconv}/include/* $out/include cp -rL ${getDev gnugrep.pcre2}/include/* $out/include mv $out/include $out/include-Libsystem # Copy coreutils, bash, etc. cp ${coreutils_}/bin/* $out/bin cp ${getBin coreutils_}/bin/* $out/bin (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) cp ${bash}/bin/bash $out/bin cp ${getBin bash}/bin/bash $out/bin ln -s bash $out/bin/sh cp ${findutils}/bin/find $out/bin cp ${findutils}/bin/xargs $out/bin cp -d ${diffutils}/bin/* $out/bin cp -d ${gnused}/bin/* $out/bin cp -d ${gnugrep}/bin/grep $out/bin cp ${gawk}/bin/gawk $out/bin cp -d ${gawk}/bin/awk $out/bin cp ${gnutar}/bin/tar $out/bin cp ${gzip}/bin/.gzip-wrapped $out/bin/gzip cp ${bzip2_.bin}/bin/bzip2 $out/bin cp ${getBin findutils}/bin/find $out/bin cp ${getBin findutils}/bin/xargs $out/bin cp -d ${getBin diffutils}/bin/* $out/bin cp -d ${getBin gnused}/bin/* $out/bin cp -d ${getBin gnugrep}/bin/grep $out/bin cp ${getBin gawk}/bin/gawk $out/bin cp -d ${getBin gawk}/bin/awk $out/bin cp ${getBin gnutar}/bin/tar $out/bin cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip cp ${getBin bzip2}/bin/bzip2 $out/bin ln -s bzip2 $out/bin/bunzip2 cp -d ${gnumake}/bin/* $out/bin cp -d ${patch}/bin/* $out/bin cp -d ${xz.bin}/bin/xz $out/bin cp ${cpio}/bin/cpio $out/bin cp -d ${getBin gnumake}/bin/* $out/bin cp -d ${getBin patch}/bin/* $out/bin cp -d ${getBin xz}/bin/xz $out/bin cp ${getBin cpio}/bin/cpio $out/bin # This used to be in-nixpkgs, but now is in the bundle # because I can't be bothered to make it partially static cp ${curl_.bin}/bin/curl $out/bin cp -d ${curl_.out}/lib/libcurl*.dylib $out/lib cp -d ${libssh2.out}/lib/libssh*.dylib $out/lib cp -d ${lib.getLib openssl}/lib/*.dylib $out/lib cp -d ${gnugrep.pcre2.out}/lib/libpcre2*.dylib $out/lib cp -d ${lib.getLib libiconv}/lib/lib*.dylib $out/lib cp -d ${lib.getLib gettext}/lib/libintl*.dylib $out/lib cp ${getBin curl_}/bin/curl $out/bin cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib cp -d ${getLib openssl}/lib/*.dylib $out/lib cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib chmod +x $out/lib/libintl*.dylib cp -d ${ncurses.out}/lib/libncurses*.dylib $out/lib cp -d ${libxml2.out}/lib/libxml2*.dylib $out/lib cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib # Copy what we need of clang cp -d ${llvmPackages.clang-unwrapped}/bin/clang* $out/bin cp -rd ${lib.getLib llvmPackages.clang-unwrapped}/lib/* $out/lib cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib cp -d ${lib.getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib '' # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11 + lib.optionalString (llvmPackages ? libcxxabi) '' cp -d ${lib.getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib '' + '' cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib cp -d ${lib.getLib llvmPackages.llvm.lib}/lib/libLLVM.dylib $out/lib cp -d ${lib.getLib libffi}/lib/libffi*.dylib $out/lib cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib mkdir -p $out/lib/darwin cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib mkdir $out/include cp -rd ${llvmPackages.libcxx.dev}/include/c++ $out/include cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include # copy .tbd assembly utils cp -d ${pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin cp -d ${lib.getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib cp -d ${getBin pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin cp -d ${getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib # copy package extraction tools cp -d ${pkgs.pbzx}/bin/pbzx $out/bin cp -d ${lib.getLib pkgs.xar}/lib/libxar*.dylib $out/lib cp -d ${pkgs.bzip2.out}/lib/libbz2*.dylib $out/lib cp -d ${getBin pkgs.pbzx}/bin/pbzx $out/bin cp -d ${getLib pkgs.xar}/lib/libxar*.dylib $out/lib cp -d ${getLib pkgs.bzip2}/lib/libbz2*.dylib $out/lib # copy sigtool cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin cp -d ${lib.getLib darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${zlib.out}/lib/libz.* $out/lib cp -d ${gmpxx.out}/lib/libgmp*.* $out/lib cp -d ${xz.out}/lib/liblzma*.* $out/lib cp -d ${getLib zlib}/lib/libz.* $out/lib cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib cp -d ${getLib xz}/lib/liblzma*.* $out/lib # Copy binutils. for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do cp ${cctools_}/bin/$i $out/bin cp ${getBin cctools_}/bin/$i $out/bin done cp -d ${lib.getLib darwin.libtapi}/lib/libtapi* $out/lib cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib cp -rd ${pkgs.darwin.CF}/Library $out ${lib.optionalString stdenv.targetPlatform.isAarch64 '' cp -rd ${pkgs.darwin.libobjc}/lib/* $out/lib/ ''} # tools needed to unpack bootstrap archive. they should not contain any # external references. we will process them like the other tools but # perform some additional checks and will not pack them into the archive. mkdir -p unpack/bin cp ${getBin bash}/bin/bash unpack/bin ln -s bash unpack/bin/sh cp ${getBin coreutils_}/bin/mkdir unpack/bin cp ${getBin gnutar_}/bin/tar unpack/bin cp ${getBin xz_}/bin/xz unpack/bin cp ${unpackScript} unpack/bootstrap-tools-unpack.sh chmod -R u+w $out # # All files copied. Perform processing to update references to point into # the archive # nuke-refs $out/bin/* chmod -R u+w $out unpack # - change nix store library paths to use @rpath/library # - if needed add an rpath containing lib/ # - strip executable rpathify() { local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true local newlib local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) local lib rpath for lib in $libs; do ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" done } case "$(dirname "$1")" in */bin) # Strip executables even further for i in $out/bin/*; do if [[ ! -L $i ]] && isMachO "$i"; then chmod +w $i ${stdenv.cc.targetPrefix}strip $i || true ${stdenv.cc.targetPrefix}strip "$i" rpath='@executable_path/../lib' ;; */lib) # the '/.' suffix is required rpath='@loader_path/.' ;; */lib/darwin) rpath='@loader_path/..' ;; *) echo unkown executable $1 >&2 exit 1 ;; esac # if shared object contains references add an rpath to lib/ if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" fi done } for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do if [[ ! -L "$i" ]]; then rpathify $i # check that linked library paths exist in $out/lib # must be run after rpathify is performed checkDeps() { local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) local lib for lib in $deps; do if [[ ! -e $out/''${lib/@rpath/lib} ]]; then echo "error: $1 missing lib for $lib" >&2 exit 1 fi done } for i in $out/bin/*; do if [[ ! -L "$i" ]] && isMachO "$i"; then ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do if [[ ! -L $i ]] && isMachO "$i"; then rpathify "$i" checkDeps "$i" fi done ${if stdenv.targetPlatform.isx86_64 then '' rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation '' else '' sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \ $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd ''} nuke-refs $out/bin/* nuke-refs $out/lib/* nuke-refs $out/lib/system/* nuke-refs $out/lib/darwin/* ${lib.optionalString stdenv.targetPlatform.isx86_64 '' nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ''} nuke-refs $out/lib/system/* nuke-refs unpack/bin/* mkdir $out/.pack mv $out/* $out/.pack mv $out/.pack $out/pack # validate that tools contain no references into the archive for tool in unpack/bin/*; do deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true) if [[ -n "$deps" ]]; then printf "error: $tool is not self contained\n$deps\n" >&2 exit 1 fi done mkdir $out/on-server dumpnar $out/pack | ${xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz cp -r unpack $out XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz ''; allowedReferences = []; Loading @@ -204,44 +296,49 @@ rec { }; }; dist = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; buildCommand = '' dist = runCommand "stdenv-bootstrap-tools" {} '' mkdir -p $out/nix-support echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products ''; }; bootstrapFiles = { tools = "${build}/pack"; bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; unpack = runCommand "unpack" { allowedReferences = []; } '' cp -r ${build}/unpack $out ''; }; bootstrapTools = derivation { inherit (stdenv.hostPlatform) system; name = "bootstrap-tools"; builder = "${bootstrapFiles.tools}/bin/bash"; builder = "${bootstrapFiles.unpack}/bin/bash"; # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly, # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the # patch-bootstrap-tools.sh for the next round of bootstrap tools. args = [ ./patch-bootstrap-tools-next.sh ]; args = [ "${bootstrapFiles.unpack}/bootstrap-tools-unpack.sh" bootstrapFiles.bootstrapTools ]; inherit (bootstrapFiles) tools; PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ]; allowedReferences = [ "out" ]; }; test = stdenv.mkDerivation { name = "test"; realBuilder = "${bootstrapTools}/bin/bash"; test = derivation { name = "test-bootstrap-tools"; inherit (stdenv.hostPlatform) system; builder = "${bootstrapTools}/bin/bash"; args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ]; PATH = lib.makeBinPath [ bootstrapTools ]; tools = bootstrapTools; buildCommand = '' "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion; # Create a pure environment where we use just what's in the bootstrap tools. export PATH=$tools/bin buildCommand = '' ls -l mkdir $out Loading @@ -268,16 +365,15 @@ rec { ${stdenv.cc.libc_dev}/lib/system \ libSystem-boot substituteInPlace libSystem-boot/libSystem.B.tbd \ --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/" sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd # End of bootstrap libSystem export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" export CC="clang $flags -rpath $tools/lib" export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib" export CC="clang $flags" export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1" # NOTE: These tests do a separate 'install' step (using cp), because # having clang write directly to the final location apparently will make Loading @@ -294,22 +390,23 @@ rec { cp hello1 $out/bin/ $out/bin/hello1 echo '#include <CoreFoundation/CoreFoundation.h>' >> hello2.c echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c cp hello2 $out/bin/ $out/bin/hello2 echo '#include <iostream>' >> hello3.cc echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc $CXX -v -o hello3 hello3.cc cp hello3 $out/bin/ $out/bin/hello3 # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked. # using -Wl,-flat_namespace is required to generate an error mkdir libtest/ ln -s $tools/lib/libc++.dylib libtest/ clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \ --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc tar xvf ${hello.src} cd hello-* # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331 am_cv_func_iconv=no ./configure --prefix=$out # hello configure detects -liconv is needed but doesn't add to the link step LDFLAGS=-liconv ./configure --prefix=$out make make install $out/bin/hello Loading
pkgs/stdenv/darwin/patch-bootstrap-tools-next.shdeleted 100644 → 0 +0 −38 Original line number Diff line number Diff line set -euo pipefail export PATH=$tools/bin cp -R $tools $out chmod -R u+w $out updateInstallName() { local path="$1" cp "$path" "$path.new" install_name_tool -id "$path" "$path.new" codesign -f -i "$(basename "$path")" -s - "$path.new" mv -f "$path.new" "$path" } find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do updateInstallName "$lib" done # Provide a gunzip script. cat > $out/bin/gunzip <<EOF #!$out/bin/sh exec $out/bin/gzip -d "\$@" EOF chmod +x $out/bin/gunzip # Provide fgrep/egrep. echo "#! $out/bin/sh" > $out/bin/egrep echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep echo "#! $out/bin/sh" > $out/bin/fgrep echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep cat >$out/bin/dsymutil << EOF #!$out/bin/sh EOF chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil