Commit 091d039b authored by Winter's avatar Winter
Browse files

prefetch-npm-deps: deduplicate dependencies when parsing lockfile 

A `package-lock.json` file can contain multiple instances of the same dependency, which caused unnecessary downloads and duplicate index entries in the generated cache.
parent b117b359

pkgs/build-support/node/fetch-npm-deps/default.nix

+6 −6
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ 
            hash = "sha256-uQmc+S+V1co1Rfc4d82PpeXjmd1UqdsG492ADQFcZGA=";

          };



          hash = "sha256-fk7L9vn8EHJsGJNMAjYZg9h0PT6dAwiahdiEeXVrMB8=";

          hash = "sha256-wca1QvxUw3OrLStfYN9Co6oVBR1LbfcNUKlDqvObps4=";

        };



        lockfileV2 = makeTest {
@@ -57,7 +57,7 @@ 
            hash = "sha256-qS29tq5QPnGxV+PU40VgMAtdwVLtLyyhG2z9GMeYtC4=";

          };



          hash = "sha256-s8SpZY/1tKZVd3vt7sA9vsqHvEaNORQBMrSyhWpj048=";

          hash = "sha256-tuEfyePwlOy2/mOPdXbqJskO6IowvAP4DWg8xSZwbJw=";

        };



        hashPrecedence = makeTest {
@@ -68,7 +68,7 @@ 
            hash = "sha256-1+0AQw9EmbHiMPA/H8OP8XenhrkhLRYBRhmd1cNPFjk=";

          };



          hash = "sha256-KRxwrEij3bpZ5hbQhX67KYpnY2cRS7u2EVZIWO1FBPM=";

          hash = "sha256-oItUls7AXcCECuyA+crQO6B0kv4toIr8pBubNwB7kAM=";

        };



        hostedGitDeps = makeTest {
@@ -79,7 +79,7 @@ 
            hash = "sha256-X9mCwPqV5yP0S2GonNvpYnLSLJMd/SUIked+hMRxDpA=";

          };



          hash = "sha256-oIM05TGHstX1D4k2K4TJ+SHB7H/tNKzxzssqf0GJwvY=";

          hash = "sha256-ri8qvYjn420ykmCC2Uy5P3jxVVrKWJG3ug/qLIGcR7o=";

        };



        linkDependencies = makeTest {
@@ -90,7 +90,7 @@ 
            hash = "sha256-6ZTBMyuyPP/63gpQugggHhKVup6OB4hZ2rmSvPJ0yEs=";

          };



          hash = "sha256-uQx8F5OXKm+fqx6hP6obVYTlQIYcJwtO52j6VQNo7Sk=";

          hash = "sha256-VzQhArHoznYSXUT7l9HkJV4yoSOmoP8eYTLel1QwmB4=";

        };



        # This package contains both hosted Git shorthand, and a bundled dependency that happens to override an existing one.
@@ -102,7 +102,7 @@ 
            hash = "sha256-1fGNxYJi1I4cXK/jinNG+Y6tPEOhP3QAqWOBEQttS9E=";

          };



          hash = "sha256-TzUFykASDjXlfmgq2bUIJjUkfLlGLUkRTVFPIYPCmyc=";

          hash = "sha256-73rLcSBgsZRJFELaKK++62hVbt1QT8JgLu2hyDSmIZE=";

        };

      };

pkgs/build-support/node/fetch-npm-deps/src/cacache.rs

+1 −1
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ impl Cache { 


        let mut file = File::options().append(true).create(true).open(index_path)?;



        write!(file, "\n{:x}\t{data}", Sha1::new().chain(&data).finalize())?;

        write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?;



        Ok(())

    }

pkgs/build-support/node/fetch-npm-deps/src/main.rs

+49 −38
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ use anyhow::{anyhow, Context}; 
use rayon::prelude::*;

use serde::Deserialize;

use std::{

    collections::HashMap,

    collections::{HashMap, HashSet},

    env, fmt, fs,

    path::Path,

    process::{self, Command},
@@ -292,14 +292,25 @@ fn main() -> anyhow::Result<()> { 
        return Ok(());

    }



    let packages = {

        let mut seen = HashSet::new();

        let mut new_packages = HashMap::new();



        for (dep, package) in packages.unwrap().drain() {

            if let (false, Some(UrlOrString::Url(resolved))) = (dep.is_empty(), &package.resolved) {

                if !seen.contains(resolved) {

                    seen.insert(resolved.clone());

                    new_packages.insert(dep, package);

                }

            }

        }



        new_packages

    };



    let cache = Cache::new(out.join("_cacache"));



    packages

        .unwrap()

        .into_par_iter()

        .filter(|(dep, _)| !dep.is_empty())

        .filter(|(_, package)| matches!(package.resolved, Some(UrlOrString::Url(_))))

        .try_for_each(|(dep, package)| {

    packages.into_par_iter().try_for_each(|(dep, package)| {

        eprintln!("{dep}");



        let mut resolved = match package.resolved {