Commit 208d7865 authored by Yureka's avatar Yureka Committed by Florian Klink
Browse files

yarn-berry: init fetcher infrastructure

parent c6c0dd59

pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch

0 → 100644
+85 −0
Original line number Diff line number Diff line 
diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts

index 9dcd02d..cf1765a 100644

--- a/packages/plugin-essentials/sources/commands/install.ts

+++ b/packages/plugin-essentials/sources/commands/install.ts

@@ -254,6 +254,7 @@ export default class YarnCommand extends BaseCommand {

       // If migrating from a v1 install, we automatically enable the node-modules linker,

       // since that's likely what the author intended to do.

       if (content?.includes(`yarn lockfile v1`)) {

+        throw new Error("Tried to use yarn-berry_3.yarnConfigHook (nixpkgs), but found a yarn v1 lockfile");

         const nmReport = await StreamReport.start({

           configuration,

           json: this.json,

diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts

index fe2a4fc..bfa8272 100644

--- a/packages/plugin-git/sources/GitFetcher.ts

+++ b/packages/plugin-git/sources/GitFetcher.ts

@@ -50,9 +50,14 @@ export class GitFetcher implements Fetcher {

   }

 

   async cloneFromRemote(locator: Locator, opts: FetchOptions) {

-    const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration);

-

     const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);

+

+    if (repoUrlParts.treeish.protocol !== "commit") {

+        throw new Error(`Missing source for git dependency ${locator.reference}`);

+    };

+

+    const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request);

+

     const packagePath = ppath.join(cloneTarget, `package.tgz` as PortablePath);

 

     await scriptUtils.prepareExternalProject(cloneTarget, packagePath, {

diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts

index 0f69423..5b21462 100644

--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts

+++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts

@@ -47,6 +47,7 @@ export class NpmSemverFetcher implements Fetcher {

   }

 

   private async fetchFromNetwork(locator: Locator, opts: FetchOptions) {

+    throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`);

     let sourceBuffer;

     try {

       sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), {

diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts

index d5e6864..374b5d6 100644

--- a/packages/yarnpkg-core/sources/Cache.ts

+++ b/packages/yarnpkg-core/sources/Cache.ts

@@ -158,6 +158,10 @@ export class Cache {

     }

   }

 

+  checkoutPath(commit: string): string {

+    return ppath.join(ppath.join(this.cwd, "../checkouts"), commit);

+  }

+

   async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise<ZipFS> } & CacheOptions): Promise<[FakeFS<PortablePath>, () => void, string | null]> {

     const mirrorPath = this.getLocatorMirrorPath(locator);

 

diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts

index b3c2c59..6b9eb2f 100644

--- a/packages/yarnpkg-core/sources/scriptUtils.ts

+++ b/packages/yarnpkg-core/sources/scriptUtils.ts

@@ -287,10 +287,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port

             // Run an install; we can't avoid it unless we inspect the

             // package.json, which I don't want to do to keep the codebase

             // clean (even if it has a slight perf cost when cloning v1 repos)

-            const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});

-            if (install.code !== 0)

-              return install.code;

-

             stdout.write(`\n`);

 

             const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr});

@@ -375,9 +371,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port

             // We can't use `npm ci` because some projects don't have npm

             // lockfiles that are up-to-date. Hopefully npm won't decide

             // to change the versions randomly.

-            const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});

-            if (install.code !== 0)

-              return install.code;

 

             const packStream = new PassThrough();

             const packPromise = miscUtils.bufferStream(packStream);

pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch

0 → 100644
+81 −0
Original line number Diff line number Diff line 
diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts

index 90ba553..ef5368c 100644

--- a/packages/plugin-essentials/sources/commands/install.ts

+++ b/packages/plugin-essentials/sources/commands/install.ts

@@ -302,6 +302,7 @@ export default class YarnCommand extends BaseCommand {

 

         for (const rule of LOCKFILE_MIGRATION_RULES) {

           if (rule.selector(lockfileLastVersion) && typeof configuration.sources.get(rule.name) === `undefined`) {

+            throw new Error(`Tried to use yarn-berry_4.yarnConfigHook (nixpkgs) which expects lockfile version 8, but found lockfile version ${lockfileLastVersion}`);

             configuration.use(`<compat>`, {[rule.name]: rule.value}, project.cwd, {overwrite: true});

             newSettings[rule.name] = rule.value;

           }

diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts

index d9f8d85..4db9f90 100644

--- a/packages/plugin-git/sources/GitFetcher.ts

+++ b/packages/plugin-git/sources/GitFetcher.ts

@@ -50,7 +50,11 @@ export class GitFetcher implements Fetcher {

   async cloneFromRemote(locator: Locator, opts: FetchOptions) {

     const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);

 

-    const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration);

+    if (repoUrlParts.treeish.protocol !== "commit") {

+        throw new Error(`Missing source for git dependency ${locator.reference}`);

+    };

+

+    const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request);

     const projectPath = ppath.resolve(cloneTarget, repoUrlParts.extra.cwd ?? PortablePath.dot);

 

     const packagePath = ppath.join(projectPath, `package.tgz`);

diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts

index 7347859..ea5767f 100644

--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts

+++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts

@@ -45,6 +45,7 @@ export class NpmSemverFetcher implements Fetcher {

   }

 

   private async fetchFromNetwork(locator: Locator, opts: FetchOptions) {

+    throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`);

     let sourceBuffer;

     try {

       sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), {

diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts

index b712ecf..c7effbc 100644

--- a/packages/yarnpkg-core/sources/Cache.ts

+++ b/packages/yarnpkg-core/sources/Cache.ts

@@ -225,6 +225,10 @@ export class Cache {

     }

   }

 

+  checkoutPath(commit: string): string {

+    return ppath.join(ppath.join(this.cwd, "../checkouts"), commit);

+  }

+

   async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise<ZipFS>} & CacheOptions): Promise<[FakeFS<PortablePath>, () => void, string | null]> {

     const mirrorPath = this.getLocatorMirrorPath(locator);

 

diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts

index 2dcd7e5..fe40fbb 100644

--- a/packages/yarnpkg-core/sources/scriptUtils.ts

+++ b/packages/yarnpkg-core/sources/scriptUtils.ts

@@ -287,10 +287,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port

             // Run an install; we can't avoid it unless we inspect the

             // package.json, which I don't want to do to keep the codebase

             // clean (even if it has a slight perf cost when cloning v1 repos)

-            const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});

-            if (install.code !== 0)

-              return install.code;

-

             stdout.write(`\n`);

 

             const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr});

@@ -375,9 +371,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port

             // We can't use `npm ci` because some projects don't have npm

             // lockfiles that are up-to-date. Hopefully npm won't decide

             // to change the versions randomly.

-            const install = await execUtils.pipevp(`npm`, [`install`, `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});

-            if (install.code !== 0)

-              return install.code;

 

             const packStream = new PassThrough();

             const packPromise = miscUtils.bufferStream(packStream);

pkgs/by-name/ya/yarn-berry/fetcher/default.nix

0 → 100644
+77 −0
Original line number Diff line number Diff line 
{

  lib,

  newScope,

  yarn-berry,

  libzip,

  zlib,

  zlib-ng,

}:



let

  variantOverlays = {

    "3" = final: {

      berryCacheVersion = "8";



      berryOfflinePatches = [ ./berry-3-offline.patch ];



      # Known good version: 1.11.3

      libzip =

        (libzip.override {

          # Known good version: 1.3.1

          zlib = zlib;

        }).overrideAttrs

          (oA: {

            patches = (oA.patches or [ ]) ++ [

              (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch")

            ];

          });

    };

    "4" = final: {

      berryCacheVersion = "10";



      berryOfflinePatches = [ ./berry-4-offline.patch ];



      # Known good version: 1.11.3

      libzip =

        (libzip.override {

          # Known good version: 2.2.4

          zlib = zlib-ng.override {

            withZlibCompat = true;

          };

        }).overrideAttrs

          (oA: {

            patches = (oA.patches or [ ]) ++ [

              (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch")

            ];

          });

    };

  };

in



lib.makeScope newScope (

  final:

  let

    berryVersion = lib.versions.major yarn-berry.version;



    err = throw ''

      Berry version ${toString berryVersion} not supported by yarn-berry-fetcher.

      Supported versions: ${lib.concatStringsSep ", " (lib.attrNames variantOverlays)}

    '';

    variantOverlay = (variantOverlays.${berryVersion} or err) final;

  in

  (

    {

      inherit yarn-berry berryVersion;



      yarn-berry-offline = final.yarn-berry.overrideAttrs (old: {

        pname = old.pname + "-offline";

        patches = (old.patches or [ ]) ++ final.berryOfflinePatches;

      });



      yarn-berry-fetcher = final.callPackage ./yarn-berry-fetcher.nix { };

      fetchYarnBerryDeps = final.callPackage ./fetch-yarn-berry-deps.nix { };

      yarnBerryConfigHook = final.callPackage ./yarn-berry-config-hook.nix { };

    }

    // variantOverlay

  )

)

pkgs/by-name/ya/yarn-berry/fetcher/fetch-yarn-berry-deps.nix

0 → 100644
+76 −0
Original line number Diff line number Diff line 
{

  lib,

  stdenv,

  yarn-berry-fetcher,

  nix-prefetch-git,

  cacert,

  berryVersion,

}:



{

  src ? null,

  hash ? "",

  sha256 ? "",

  ...

}@args:



let

  hash_ =

    if hash != "" then

      {

        outputHashAlgo = null;

        outputHash = hash;

      }

    else if sha256 != "" then

      {

        outputHashAlgo = "sha256";

        outputHash = sha256;

      }

    else

      {

        outputHashAlgo = "sha256";

        outputHash = lib.fakeSha256;

      };

in



stdenv.mkDerivation (

  {

    # The name is fixed as to not produce multiple store paths with the same content

    name = "offline";



    dontUnpack = src == null;

    dontInstall = true;



    nativeBuildInputs = [

      yarn-berry-fetcher

      nix-prefetch-git

      cacert

    ];



    buildPhase = ''

      runHook preBuild



      yarnLock=''${yarnLock:=$PWD/yarn.lock}

      yarn-berry-fetcher fetch $yarnLock $missingHashes



      runHook postBuild

    '';



    outputHashMode = "recursive";



    passthru = {

      inherit berryVersion;

    };

  }

  // hash_

  // (removeAttrs args (

    [

      "name"

      "pname"

      "version"

      "hash"

      "sha256"

    ]

    ++ (lib.optional (src == null) "src")

  ))

)

pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.nix

0 → 100644
+24 −0
Original line number Diff line number Diff line 
{

  makeSetupHook,

  yarn-berry-offline,

  srcOnly,

  nodejs,

  diffutils,

}:



makeSetupHook {

  name = "yarn-berry-config-hook";

  substitutions = {

    # Specify `diff` by abspath to ensure that the user's build

    # inputs do not cause us to find the wrong binaries.

    diff = "${diffutils}/bin/diff";



    yarn_offline = "${yarn-berry-offline}/bin/yarn";



    nodeSrc = srcOnly nodejs;

    nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";

  };

  meta = {

    description = "Install nodejs dependencies from an offline yarn cache produced by fetchYarnDeps";

  };

} ./yarn-berry-config-hook.sh