Unverified Commit ebd3b37e authored by David McFarland's avatar David McFarland Committed by GitHub
Browse files

Merge pull request #327651 from corngood/dotnet-unpacked-packages

dotnet: use unpacked packages in store
parents c089f552 d3ca5027
Loading
Loading
Loading
Loading
+4 −4
Original line number Diff line number Diff line
@@ -98,13 +98,13 @@ To package Dotnet applications, you can use `buildDotnetModule`. This has simila
For more detail about managing the `deps.nix` file, see [Generating and updating NuGet dependencies](#generating-and-updating-nuget-dependencies)
:::

* `packNupkg` is used to pack project as a `nupkg`, and installs it to `$out/share`. If set to `true`, the derivation can be used as a dependency for another dotnet project by adding it to `projectReferences`.
* `projectReferences` can be used to resolve `ProjectReference` project items. Referenced projects can be packed with `buildDotnetModule` by setting the `packNupkg = true` attribute and passing a list of derivations to `projectReferences`. Since we are sharing referenced projects as NuGets they must be added to csproj/fsproj files as `PackageReference` as well.
* `packNupkg` is used to pack project as a `nupkg`, and installs it to `$out/share`. If set to `true`, the derivation can be used as a dependency for another dotnet project by adding it to `buildInputs`.
* `buildInputs` can be used to resolve `ProjectReference` project items. Referenced projects can be packed with `buildDotnetModule` by setting the `packNupkg = true` attribute and passing a list of derivations to `buildInputs`. Since we are sharing referenced projects as NuGets they must be added to csproj/fsproj files as `PackageReference` as well.
 For example, your project has a local dependency:
 ```xml
     <ProjectReference Include="../foo/bar.fsproj" />
 ```
 To enable discovery through `projectReferences` you would need to add:
 To enable discovery through `buildInputs` you would need to add:
 ```xml
     <ProjectReference Include="../foo/bar.fsproj" />
     <PackageReference Include="bar" Version="*" Condition=" '$(ContinuousIntegrationBuild)'=='true' "/>
@@ -143,7 +143,7 @@ in buildDotnetModule rec {
  projectFile = "src/project.sln";
  nugetDeps = ./deps.nix; # see "Generating and updating NuGet dependencies" section for details

  projectReferences = [ referencedProject ]; # `referencedProject` must contain `nupkg` in the folder structure.
  buildInputs = [ referencedProject ]; # `referencedProject` must contain `nupkg` in the folder structure.

  dotnet-sdk = dotnetCorePackages.sdk_6_0;
  dotnet-runtime = dotnetCorePackages.runtime_6_0;
+1 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ buildDotnetModule (args // {
    nugetDeps = { fetchNuGet }: [
      (fetchNuGet { pname = nugetName; inherit version; sha256 = nugetSha256; hash = nugetHash; })
    ] ++ (nugetDeps fetchNuGet);
    installable = true;
  };

  dotnetGlobalTool = true;
+53 −83
Original line number Diff line number Diff line
@@ -4,20 +4,17 @@
, callPackage
, substituteAll
, writeShellScript
, srcOnly
, linkFarmFromDrvs
, symlinkJoin
, makeWrapper
, dotnetCorePackages
, mkNugetSource
, mkNugetDeps
, nuget-to-nix
, cacert
, coreutils
, runtimeShellPackage
, unzip
, yq
, nix
}:

{ name ? "${args.pname}-${args.version}"
{ name ? "${_args.pname}-${_args.version}"
, pname ? name
, enableParallelBuilding ? true
, doCheck ? false
@@ -80,16 +77,18 @@
  # Whether to use an alternative wrapper, that executes the application DLL using the dotnet runtime from the user environment. `dotnet-runtime` is provided as a default in case no .NET is installed
  # This is useful for .NET tools and applications that may need to run under different .NET runtimes
, useDotnetFromEnv ? false
  # Whether to explicitly enable UseAppHost when building. This is redundant if useDotnetFromEnv is enabledz
  # Whether to explicitly enable UseAppHost when building. This is redundant if useDotnetFromEnv is enabled
, useAppHost ? true
  # The dotnet SDK to use.
, dotnet-sdk ? dotnetCorePackages.sdk_6_0
  # The dotnet runtime to use.
, dotnet-runtime ? dotnetCorePackages.runtime_6_0
, ...
} @ args:
} @ _args:

let
  args = removeAttrs _args [ "nugetDeps" ];

  projectFiles =
    lib.optionals (projectFile != null) (lib.toList projectFile);
  testProjectFiles =
@@ -104,11 +103,6 @@ let
    inherit dotnet-sdk dotnet-runtime;
  }) dotnetConfigureHook dotnetBuildHook dotnetCheckHook dotnetInstallHook dotnetFixupHook;

  localDeps =
    if (projectReferences != [ ])
    then linkFarmFromDrvs "${name}-project-references" projectReferences
    else null;

  _nugetDeps =
    if (nugetDeps != null) then
      if lib.isDerivation nugetDeps
@@ -119,41 +113,21 @@ let
      }
    else throw "Defining the `nugetDeps` attribute is required, as to lock the NuGet dependencies. This file can be generated by running the `passthru.fetch-deps` script.";

  # contains the actual package dependencies
  dependenciesSource = mkNugetSource {
    name = "${name}-dependencies-source";
    description = "Nuget source with the dependencies for ${name}";
    deps = [ _nugetDeps ] ++ lib.optional (localDeps != null) localDeps;
  };

  # this contains all the nuget packages that are implicitly referenced by the dotnet
  # build system. having them as separate deps allows us to avoid having to regenerate
  # a packages dependencies when the dotnet-sdk version changes
  sdkDeps = lib.lists.flatten [ dotnet-sdk.packages ];

  sdkSource = let
    version = dotnet-sdk.version or (lib.concatStringsSep "-" dotnet-sdk.versions);
  in mkNugetSource {
    name = "dotnet-sdk-${version}-source";
    deps = sdkDeps;
  };

  nuget-source = symlinkJoin {
    name = "${name}-nuget-source";
    paths = [ dependenciesSource sdkSource ];
  };

  nugetDepsFile = _nugetDeps.sourceFile;

  inherit (dotnetCorePackages) systemToDotnetRid;
in
stdenvNoCC.mkDerivation (args // {
stdenvNoCC.mkDerivation (finalAttrs: args // {
  dotnetInstallPath = installPath;
  dotnetExecutables = executables;
  dotnetBuildType = buildType;
  dotnetProjectFiles = projectFiles;
  dotnetTestProjectFiles = testProjectFiles;
  dotnetDisabledTests = disabledTests;
  dotnetRuntimeId = runtimeId;
  nugetSource = nuget-source;
  dotnetRuntimeIds = lib.singleton (
    if runtimeId != null
    then runtimeId
    else systemToDotnetRid stdenvNoCC.hostPlatform.system);
  dotnetRuntimeDeps = map lib.getLib runtimeDeps;
  dotnetSelfContainedBuild = selfContainedBuild;
  dotnetUseAppHost = useAppHost;
@@ -169,8 +143,15 @@ stdenvNoCC.mkDerivation (args // {
    cacert
    makeWrapper
    dotnet-sdk
    unzip
    yq
  ];

  buildInputs = args.buildInputs or [] ++ [
    dotnet-sdk.packages
    _nugetDeps
  ] ++ projectReferences;

  # Parse the version attr into a format acceptable for the Version msbuild property
  # The actual version attr is saved in InformationalVersion, which accepts an arbitrary string
  versionForDotnet = if !(lib.hasAttr "version" args) || args.version == null
@@ -202,19 +183,21 @@ stdenvNoCC.mkDerivation (args // {
  propagatedSandboxProfile = toString dotnet-runtime.__propagatedSandboxProfile;

  passthru = {
    inherit nuget-source;
    nugetDeps = _nugetDeps;
  } // lib.optionalAttrs (!lib.isDerivation nugetDeps) {
    fetch-deps = let
      flagsList = dotnetFlags ++ dotnetRestoreFlags;
      flags = lib.concatStringsSep " " (map lib.escapeShellArg flagsList);
      disableParallel =
        lib.optionalString (!enableParallelBuilding) "--disable-parallel";
      runtimeIdsList = if runtimeId != null then
        [ runtimeId ]
      else
        map (system: dotnetCorePackages.systemToDotnetRid system) platforms;
      runtimeIds =
        lib.concatStringsSep " " (map lib.escapeShellArg runtimeIdsList);
      pkg = finalAttrs.finalPackage.overrideAttrs (old: {
        buildInputs = lib.remove _nugetDeps old.buildInputs;
        keepNugetConfig = true;
      } // lib.optionalAttrs (runtimeId == null) {
        dotnetRuntimeIds = map (system: systemToDotnetRid system) platforms;
      });

      drv = builtins.unsafeDiscardOutputDependency pkg.drvPath;

      innerScript = substituteAll {
        src = ./fetch-deps.sh;
        isExecutable = true;
        defaultDepsFile =
          # Wire in the nugetDeps file such that running the script with no args
          # runs it agains the correct deps file by default.
@@ -226,26 +209,13 @@ stdenvNoCC.mkDerivation (args // {
            toString nugetDepsFile
          else
            ''$(mktemp -t "${pname}-deps-XXXXXX.nix")'';
      storeSrc = srcOnly args;
      projectFileStr = lib.escapeShellArgs projectFiles;
      testProjectFileStr = lib.escapeShellArgs testProjectFiles;
      path = lib.makeBinPath [
        coreutils
        runtimeShellPackage
        dotnet-sdk
        (nuget-to-nix.override { inherit dotnet-sdk; })
      ];
      dotnetSdkPath = toString dotnet-sdk;
      excludedSources =
        lib.concatStringsSep " " (map lib.escapeShellArg sdkDeps);
    in substituteAll {
      name = "fetch-deps";
      src = ./fetch-deps.sh;
      isExecutable = true;
      inherit pname defaultDepsFile runtimeIds storeSrc flags disableParallel
      projectFileStr testProjectFileStr path dotnetSdkPath excludedSources
      runtimeShell;
        nugetToNix = (nuget-to-nix.override { inherit dotnet-sdk; });
      };

      in writeShellScript "${name}-fetch-deps" ''
        NIX_BUILD_SHELL="${runtimeShell}" exec ${nix}/bin/nix-shell \
          --pure --run 'source "${innerScript}"' "${drv}"
      '';
  } // args.passthru or { };

  meta = (args.meta or { }) // { inherit platforms; };
+13 −102
Original line number Diff line number Diff line
#! @runtimeShell@
# shellcheck shell=bash
set -euo pipefail
set -e

export PATH="@path@"
tmp=$(mktemp -d)
trap 'chmod -R +w "$tmp" && rm -fr "$tmp"' EXIT

for arg in "$@"; do
    case "$arg" in
        --keep-sources|-k)
            keepSources=1
            shift
            ;;
        --help|-h)
            echo "usage: $0 [--keep-sources] [--help] <output path>"
            echo "    <output path>   The path to write the lockfile to. A temporary file is used if this is not set"
            echo "    --keep-sources  Dont remove temporary directories upon exit, useful for debugging"
            echo "    --help          Show this help message"
            exit
            ;;
    esac
done
HOME=$tmp/.home
cd "$tmp"

if [[ ${TMPDIR:-} == /run/user/* ]]; then
   # /run/user is usually a tmpfs in RAM, which may be too small
   # to store all downloaded dotnet packages
   unset TMPDIR
fi

export tmp=$(mktemp -td "deps-@pname@-XXXXXX")
HOME=$tmp/home

exitTrap() {
    test -n "${ranTrap-}" && return
    ranTrap=1

    if test -n "${keepSources-}"; then
        echo -e "Path to the source: $tmp/src\nPath to the fake home: $tmp/home"
    else
        rm -rf "$tmp"
    fi

    # Since mktemp is used this will be empty if the script didnt succesfully complete
    if ! test -s "$depsFile"; then
      rm -rf "$depsFile"
    fi
}

trap exitTrap EXIT INT TERM

dotnetRestore() {
    local -r project="${1-}"
    local -r rid="$2"

    dotnet restore ${project-} \
        -p:ContinuousIntegrationBuild=true \
        -p:Deterministic=true \
        --packages "$tmp/nuget_pkgs" \
        --runtime "$rid" \
        --no-cache \
        --force \
        @disableParallel@ \
        @flags@
}

declare -a projectFiles=( @projectFileStr@ )
declare -a testProjectFiles=( @testProjectFileStr@ )

export DOTNET_NOLOGO=1
export DOTNET_CLI_TELEMETRY_OPTOUT=1
phases="
    ${prePhases[*]:-}
    unpackPhase
    patchPhase
    ${preConfigurePhases[*]:-}
    configurePhase
" genericBuild

depsFile=$(realpath "${1:-@defaultDepsFile@}")
echo Will write lockfile to "$depsFile"
mkdir -p "$tmp/nuget_pkgs"

storeSrc="@storeSrc@"
src=$tmp/src
cp -rT "$storeSrc" "$src"
chmod -R +w "$src"

cd "$src"
echo "Restoring project..."

"@dotnetSdkPath@/bin/dotnet" tool restore
cp -r $HOME/.nuget/packages/* $tmp/nuget_pkgs || true

runtimeIds=(@runtimeIds@)

for rid in "${runtimeIds[@]}"; do
    (( ${#projectFiles[@]} == 0 )) && dotnetRestore "" "$rid"

    for project in ${projectFiles[@]-} ${testProjectFiles[@]-}; do
        dotnetRestore "$project" "$rid"
    done
done
# Second copy, makes sure packages restored by ie. paket are included
cp -r $HOME/.nuget/packages/* $tmp/nuget_pkgs || true

echo "Succesfully restored project"

echo "Writing lockfile..."

excluded_sources=( @excludedSources@ )
for excluded_source in ${excluded_sources[@]}; do
  ls "$excluded_source" >> "$tmp/excluded_list"
done
tmpFile="$tmp"/deps.nix
echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n" > "$tmpFile"
nuget-to-nix "$tmp/nuget_pkgs" "$tmp/excluded_list" >> "$tmpFile"
@nugetToNix@/bin/nuget-to-nix "$NUGET_PACKAGES" >> "$tmpFile"
mv "$tmpFile" "$depsFile"
echo "Succesfully wrote lockfile to $depsFile"
+2 −12
Original line number Diff line number Diff line
@@ -5,20 +5,16 @@
, zlib
, openssl
, makeSetupHook
, dotnetCorePackages
, zip
  # Passed from ../default.nix
, dotnet-sdk
, dotnet-runtime
}:
let
  runtimeId = dotnetCorePackages.systemToDotnetRid stdenv.hostPlatform.system;
in
{
  dotnetConfigureHook = makeSetupHook
    {
      name = "dotnet-configure-hook";
      substitutions = {
        runtimeId = lib.escapeShellArg runtimeId;
        dynamicLinker = "${stdenv.cc}/nix-support/dynamic-linker";
        libPath = lib.makeLibraryPath [
          stdenv.cc.cc.lib
@@ -34,18 +30,12 @@ in
  dotnetBuildHook = makeSetupHook
    {
      name = "dotnet-build-hook";
      substitutions = {
        runtimeId = lib.escapeShellArg runtimeId;
      };
    }
    ./dotnet-build-hook.sh;

  dotnetCheckHook = makeSetupHook
    {
      name = "dotnet-check-hook";
      substitutions = {
        runtimeId = lib.escapeShellArg runtimeId;
      };
    }
    ./dotnet-check-hook.sh;

@@ -53,7 +43,7 @@ in
    {
      name = "dotnet-install-hook";
      substitutions = {
        runtimeId = lib.escapeShellArg runtimeId;
        inherit zip;
      };
    }
    ./dotnet-install-hook.sh;
Loading