Unverified Commit 554e2412 authored by Lily Foster's avatar Lily Foster
Browse files

prefetch-npm-deps: read url bodies within the retry loop

parent a6df5a77

pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs

+4 −10
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ use rayon::prelude::*; 
use serde_json::{Map, Value};

use std::{

    fs,

    io::{self, Read},

    io::Write,

    process::{Command, Stdio},

};

use tempfile::{tempdir, TempDir};
@@ -106,7 +106,7 @@ impl Package { 


        let specifics = match get_hosted_git_url(&resolved)? {

            Some(hosted) => {

                let mut body = util::get_url_with_retry(&hosted)?;

                let body = util::get_url_body_with_retry(&hosted)?;



                let workdir = tempdir()?;
@@ -120,7 +120,7 @@ impl Package { 
                    .stdin(Stdio::piped())

                    .spawn()?;



                io::copy(&mut body, &mut cmd.stdin.take().unwrap())?;

                cmd.stdin.take().unwrap().write_all(&body)?;



                let exit = cmd.wait()?;
@@ -154,13 +154,7 @@ impl Package { 


    pub fn tarball(&self) -> anyhow::Result<Vec<u8>> {

        match &self.specifics {

            Specifics::Registry { .. } => {

                let mut body = Vec::new();



                util::get_url_with_retry(&self.url)?.read_to_end(&mut body)?;



                Ok(body)

            }

            Specifics::Registry { .. } => Ok(util::get_url_body_with_retry(&self.url)?),

            Specifics::Git { workdir } => Ok(Command::new("tar")

                .args([

                    "--sort=name",

pkgs/build-support/node/fetch-npm-deps/src/util.rs

+18 −10
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ use isahc::{ 
    Body, Request, RequestExt,

};

use serde_json::{Map, Value};

use std::{env, path::Path};

use std::{env, io::Read, path::Path};

use url::Url;



pub fn get_url(url: &Url) -> Result<Body, isahc::Error> {
@@ -28,7 +28,7 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { 
    if let Some(host) = url.host_str() {

        if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") {

            if let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens) {

                if let Some(token) = tokens.get(host).and_then(|val| val.as_str()) {

                if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) {

                    request = request.header("Authorization", format!("Bearer {token}"));

                }

            }
@@ -38,9 +38,17 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { 
    Ok(request.body(())?.send()?.into_body())

}



pub fn get_url_with_retry(url: &Url) -> Result<Body, isahc::Error> {

pub fn get_url_body_with_retry(url: &Url) -> Result<Vec<u8>, isahc::Error> {

    retry(ExponentialBackoff::default(), || {

        get_url(url).map_err(|err| {

        get_url(url)

            .and_then(|mut body| {

                let mut buf = Vec::new();



                body.read_to_end(&mut buf)?;



                Ok(buf)

            })

            .map_err(|err| {

                if err.is_network() || err.is_timeout() {

                    backoff::Error::transient(err)

                } else {