Unverified Commit 363fc3ca authored by David McFarland's avatar David McFarland Committed by GitHub
Browse files

Merge pull request #309409 from corngood/dotnet-aot 

dotnet: fix aot compilation
parents 0101bf73 f535b0c6

pkgs/development/compilers/dotnet/build-dotnet.nix

+22 −5
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ assert if type == "sdk" then packages != null else true; 
, libuuid

, zlib

, libkrb5

, openssl

, curl

, lttng-ust_2_12

, testers
@@ -51,6 +52,21 @@ let 
  sigtool = callPackage ./sigtool.nix {};

  signAppHost = callPackage ./sign-apphost.nix {};



  hasILCompiler =

    lib.versionAtLeast version (if targetRid == "osx-arm64" then "8" else "7");



  extraTargets = writeText "extra.targets" (''

    <Project>

  '' + lib.optionalString hasILCompiler ''

      <ItemGroup>

        <CustomLinkerArg Include="-Wl,-rpath,'${lib.makeLibraryPath [ icu zlib openssl ]}'" />

      </ItemGroup>

  '' + lib.optionalString stdenv.isDarwin ''

      <Import Project="${signAppHost}" />

  '' + ''

    </Project>

  '');



in

mkCommon type rec {

  inherit pname version;
@@ -70,6 +86,7 @@ mkCommon type rec { 
    icu

    libkrb5

    curl

    xmlstarlet

  ] ++ lib.optional stdenv.isLinux lttng-ust_2_12;



  src = fetchurl (
@@ -79,15 +96,15 @@ mkCommon type rec { 


  sourceRoot = ".";



  postPatch = if type == "sdk" && stdenv.isDarwin then ''

  postPatch = if type == "sdk" then (''

    xmlstarlet ed \

      --inplace \

      -s //_:Project -t elem -n Import \

      -i \$prev -t attr -n Project -v "${signAppHost}" \

      -i \$prev -t attr -n Project -v "${extraTargets}" \

      sdk/*/Sdks/Microsoft.NET.Sdk/targets/Microsoft.NET.Sdk.targets



  '' + lib.optionalString stdenv.isDarwin ''

    codesign --remove-signature packs/Microsoft.NETCore.App.Host.osx-*/*/runtimes/osx-*/native/{apphost,singlefilehost}

  '' else null;

  '') else null;



  dontPatchELF = true;

  noDumpEnvVars = true;
@@ -135,7 +152,7 @@ mkCommon type rec { 
  '';



  passthru = {

    inherit icu;

    inherit icu hasILCompiler;

  } // lib.optionalAttrs (type == "sdk") {

    packages = mkNugetDeps {

      name = "${pname}-${version}-deps";

pkgs/development/compilers/dotnet/common.nix

+52 −16
Original line number Diff line number Diff line 
# TODO: switch to stdenvNoCC

{ stdenv

, stdenvNoCC

, lib

, writeText

, testers

, runCommand

, runCommandWith

, expect

, curl

, installShellFiles

, callPackage

, zlib

, swiftPackages

, darwin

, icu

}: type: args: stdenv.mkDerivation (finalAttrs: args // {

  doInstallCheck = true;
@@ -43,9 +50,11 @@ 
      mkDotnetTest =

        {

          name,

          stdenv ? stdenvNoCC,

          template,

          usePackageSource ? false,

          build,

          buildInputs ? [],

          # TODO: use correct runtimes instead of sdk

          runtime ? finalAttrs.finalPackage,

          runInputs ? [],
@@ -54,19 +63,20 @@ 
        }:

        let

          sdk = finalAttrs.finalPackage;

          built = runCommand "dotnet-test-${name}" {

            buildInputs = [ sdk ];

          built = runCommandWith  {

            name = "dotnet-test-${name}";

            inherit stdenv;

            derivationArgs = {

              buildInputs = [ sdk ] ++ buildInputs;

              # make sure ICU works in a sandbox

            propagatedSandboxProfile = toString sdk.__propagatedSandboxProfile + ''

              (allow network-inbound (local ip))

              (allow mach-lookup (global-name "com.apple.FSEvents"))

            '';

              propagatedSandboxProfile = toString sdk.__propagatedSandboxProfile;

            };

          } (''

            HOME=$PWD/.home

            dotnet new nugetconfig

            dotnet nuget disable source nuget

          '' + lib.optionalString usePackageSource ''

            dotnet nuget add source ${finalAttrs.finalPackage.packages}

            dotnet nuget add source ${sdk.packages}

          '' + ''

            dotnet new ${template} -n test -o .

          '' + build);
@@ -96,6 +106,8 @@ 
        [[ "$output" =~ Hello,?\ World! ]] && touch "$out"

      '';



      patchNupkgs = callPackage ./patch-nupkgs.nix {};



    in {

      version = testers.testVersion ({

        package = finalAttrs.finalPackage;
@@ -103,7 +115,7 @@ 
        command = "dotnet --info";

      });

    }

    // lib.optionalAttrs (type == "sdk") {

    // lib.optionalAttrs (type == "sdk") ({

      console = mkDotnetTest {

        name = "console";

        template = "console";
@@ -113,8 +125,8 @@ 
      publish = mkDotnetTest {

        name = "publish";

        template = "console";

        build = "dotnet publish -o $out";

        run = checkConsoleOutput "$src/test";

        build = "dotnet publish -o $out/bin";

        run = checkConsoleOutput "$src/bin/test";

      };



      self-contained = mkDotnetTest {
@@ -130,20 +142,20 @@ 
        name = "single-file";

        template = "console";

        usePackageSource = true;

        build = "dotnet publish --use-current-runtime -p:PublishSingleFile=true -o $out";

        build = "dotnet publish --use-current-runtime -p:PublishSingleFile=true -o $out/bin";

        runtime = null;

        run = checkConsoleOutput "$src/test";

        run = checkConsoleOutput "$src/bin/test";

      };



      web = mkDotnetTest {

        name = "web";

        template = "web";

        build = "dotnet publish -o $out";

        build = "dotnet publish -o $out/bin";

        runInputs = [ expect curl ];

        run = ''

          expect <<"EOF"

            set status 1

            spawn $env(src)/test

            spawn $env(src)/bin/test

            proc abort { } { exit 2 }

            expect_before default abort

            expect -re {Now listening on: ([^\r]+)\r} {
@@ -165,6 +177,30 @@ 
        '';

        runAllowNetworking = true;

      };

    } // args.passthru.tests or {};

    } // lib.optionalAttrs finalAttrs.finalPackage.hasILCompiler {

      aot = mkDotnetTest {

        name = "aot";

        stdenv = if stdenv.isDarwin then swiftPackages.stdenv else stdenv;

        template = "console";

        usePackageSource = true;

        buildInputs =

          [ patchNupkgs

            zlib

          ] ++ lib.optional stdenv.isDarwin (with darwin; with apple_sdk.frameworks; [

            swiftPackages.swift

            Foundation

            CryptoKit

            GSS

            ICU

          ]);

        build = ''

          dotnet restore -p:PublishAot=true

          patch-nupkgs .home/.nuget/packages

          dotnet publish -p:PublishAot=true -o $out/bin

        '';

        runtime = null;

        run = checkConsoleOutput "$src/bin/test";

      };

    }) // args.passthru.tests or {};

  } // args.passthru or {};

})

pkgs/development/compilers/dotnet/packages.nix

+2 −0
Original line number Diff line number Diff line
@@ -37,6 +37,8 @@ in { 


    passthru = {

      inherit (vmr) icu targetRid;

      # ilcompiler is currently broken: https://github.com/dotnet/source-build/issues/1215

      hasILCompiler = false;

    };



    meta = vmr.meta // {

pkgs/development/compilers/dotnet/patch-nupkgs.nix

+14 −2
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ let 
    openssl

  ] ++ lib.optional stdenv.isLinux lttng-ust_2_12);



in writeShellScriptBin "patch-nupkgs" ''

in writeShellScriptBin "patch-nupkgs" (''

  set -euo pipefail

  shopt -s nullglob

  isELF() {
@@ -71,4 +71,16 @@ in writeShellScriptBin "patch-nupkgs" '' 
    touch .nix-patched

    popd

  done

''
 
'' + lib.optionalString stdenv.isDarwin ''

  for x in microsoft.dotnet.ilcompiler/*; do

    # .nupkg.metadata is written last, so we know the packages is complete

    [[ -d "$x" ]] && [[ -f "$x"/.nupkg.metadata ]] \

      && [[ ! -f "$x"/.nix-patched ]] || continue

    echo "Patching package $x"

    pushd "$x"

    sed -i 's: -no_code_signature_warning::g' build/Microsoft.NETCore.Native.targets

    sed -i 's:Include="-ld_classic"::g' build/Microsoft.NETCore.Native.Unix.targets

    touch .nix-patched

    popd

  done

'')

pkgs/development/compilers/dotnet/update.sh

+228 −213
Original line number Diff line number Diff line
@@ -38,12 +38,14 @@ platform_sources () { 
    )



    echo "srcs = {"

    local kv

    for kv in "${platforms[@]}"; do

        local nix_platform=${kv%% *}

        local ms_platform=${kv##* }

        local url hash



    local url=$(release_platform_attr "$release_files" "$ms_platform" url)

    local hash=$(release_platform_attr "$release_files" "$ms_platform" hash)

        url=$(release_platform_attr "$release_files" "$ms_platform" url)

        hash=$(release_platform_attr "$release_files" "$ms_platform" hash)



        [[ -z "$url" || -z "$hash" ]] && continue

        echo "      $nix_platform = {
@@ -55,15 +57,13 @@ platform_sources () { 
}



generate_package_list() {

    local version pkgs nuget_url

    version="$1"

    local version="$1"

    shift

    pkgs=( "$@" )

    local pkgs=( "$@" ) nuget_url pkg url hash



    nuget_url="$(curl -f "https://api.nuget.org/v3/index.json" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')"



    for pkg in "${pkgs[@]}"; do

        local url hash

        url="${nuget_url}${pkg,,}/${version,,}/${pkg,,}.${version,,}.nupkg"

        hash="$(nix-prefetch-url "$url")"

        if [[ -z "$hash" ]]; then
@@ -76,8 +76,10 @@ generate_package_list() { 
}



version_older () {

    cur_version=$1

    max_version=$2

    local cur_version=$1

    local max_version=$2

    local result



    result=$(nix-instantiate -I ../../../../. \

        --eval -E "(import ../../../../. {}).lib.versionOlder \"$cur_version\" \"$max_version\"")

    if [[ "$result" == "true" ]]; then
@@ -260,6 +262,7 @@ sdk_packages () { 
    # These packages were removed on .NET 9

    if ! version_older "$version" "9"; then

        local newpkgs=()

        local pkg

        for pkg in "${pkgs[@]}"; do

            [[ "$pkg" = *Microsoft.NETCore.DotNetHost* ]] || newpkgs+=("$pkg")

        done
@@ -282,6 +285,7 @@ sdk_packages () { 
    # Packages that only apply to .NET 7 and up

    if ! version_older "$version" "7"; then

        pkgs+=(

            "Microsoft.DotNet.ILCompiler"

            "runtime.linux-arm64.Microsoft.DotNet.ILCompiler"

            "runtime.linux-musl-arm64.Microsoft.DotNet.ILCompiler"

            "runtime.linux-musl-x64.Microsoft.DotNet.ILCompiler"
@@ -296,6 +300,7 @@ sdk_packages () { 
    if ! version_older "$version" "8"; then

        pkgs+=(

            "Microsoft.NET.ILLink.Tasks"

            "runtime.osx-arm64.Microsoft.DotNet.ILCompiler"

        )

    fi
@@ -303,6 +308,7 @@ sdk_packages () { 
}



main () {

    local pname

    pname=$(basename "$0")

    if [[ ! "$*" =~ ^.*[0-9]{1,}\.[0-9]{1,}.*$ ]]; then

        echo "Usage: $pname [sem-versions]
@@ -317,7 +323,7 @@ Examples: 


    for sem_version in "$@"; do

        echo "Generating ./versions/${sem_version}.nix"

    patch_specified=false

        local patch_specified=false

        # Check if a patch was specified as an argument.

        # If so, generate file for the specific version.

        # If only x.y version was provided, get the latest patch
@@ -330,19 +336,23 @@ Examples: 


        # Make sure the x.y version is properly passed to .NET release metadata url.

        # Then get the json file and parse it to find the latest patch release.

        local major_minor content major_minor_patch

        major_minor=$(sed 's/^\([0-9]*\.[0-9]*\).*$/\1/' <<< "$sem_version")

        content=$(curl -sL https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/"$major_minor"/releases.json)

        major_minor_patch=$([ "$patch_specified" == true ] && echo "$sem_version" || jq -r '."latest-release"' <<< "$content")

    major_minor_underscore=${major_minor/./_}

        local major_minor_underscore=${major_minor/./_}



        local release_content aspnetcore_version runtime_version

        release_content=$(release "$content" "$major_minor_patch")

        aspnetcore_version=$(jq -r '."aspnetcore-runtime".version' <<< "$release_content")

        runtime_version=$(jq -r '.runtime.version' <<< "$release_content")

        local -a sdk_versions

        mapfile -t sdk_versions < <(jq -r '.sdks[] | .version' <<< "$release_content" | sort -rn)



        # If patch was not specified, check if the package is already the latest version

        # If it is, exit early

        if [ "$patch_specified" == false ] && [ -f "./versions/${sem_version}.nix" ]; then

            local current_version

            current_version=$(nix-instantiate --eval -E "(import ./versions/${sem_version}.nix { \

            buildAspNetCore = { ... }: {}; \

            buildNetSdk = { ... }: {}; \
@@ -355,15 +365,19 @@ Examples: 
            fi

        fi



        local aspnetcore_files runtime_files

        aspnetcore_files="$(release_files "$release_content" .\"aspnetcore-runtime\")"

        runtime_files="$(release_files "$release_content" .runtime)"



        local channel_version support_phase

        channel_version=$(jq -r '."channel-version"' <<< "$content")

        support_phase=$(jq -r '."support-phase"' <<< "$content")



        local aspnetcore_sources runtime_sources

        aspnetcore_sources="$(platform_sources "$aspnetcore_files")"

        runtime_sources="$(platform_sources "$runtime_files")"



        local sdk_packages aspnetcore_packages

        sdk_packages="$(sdk_packages "${runtime_version}")"

        aspnetcore_packages="$(aspnetcore_packages "${aspnetcore_version}")"
@@ -392,19 +406,20 @@ in rec { 
    $runtime_sources

  };" > "${result}"



    declare -A feature_bands

        local -A feature_bands

        unset latest_sdk



        for sdk_version in "${sdk_versions[@]}"; do

      sdk_base_version=${sdk_version%-*}

      feature_band=${sdk_base_version:0:-2}xx

            local sdk_base_version=${sdk_version%-*}

            local feature_band=${sdk_base_version:0:-2}xx

            # sometimes one release has e.g. both 8.0.202 and 8.0.203

            [[ ! ${feature_bands[$feature_band]+true} ]] || continue

            feature_bands[$feature_band]=$sdk_version

            local sdk_files sdk_sources

            sdk_files="$(release_files "$release_content" ".sdks[] | select(.version == \"$sdk_version\")")"

            sdk_sources="$(platform_sources "$sdk_files")"

      sdk_attrname=sdk_${feature_band//./_}

      [[ -v latest_sdk ]] || latest_sdk=$sdk_attrname

            local sdk_attrname=sdk_${feature_band//./_}

            [[ -v latest_sdk ]] || local latest_sdk=$sdk_attrname



            echo "

  $sdk_attrname = buildNetSdk {