Commit c4402bd4 authored by Wohlgemuth, Jason's avatar Wohlgemuth, Jason
Browse files

feat: Improve GitHub bucket support up to pagination

parent 0353c722

acorn-lib/src/lib.rs

+13 −8
Original line number Diff line number Diff line
@@ -250,7 +250,8 @@ impl Bucket { 
    }

    fn tree(&self, directory: &str, page: u32) -> eyre::Result<reqwest::blocking::Response, reqwest::Error> {

        let url = self.tree_url(directory, page);

        reqwest::blocking::get(url.unwrap_or_default())

        let client = Client::new();

        client.get(url.unwrap_or_default()).header(USER_AGENT, "rust-web-api-client").send()

    }

    fn tree_url(&self, directory: &str, page: u32) -> Option<String> {

        match &self.code_repository {
@@ -263,7 +264,7 @@ impl Bucket { 
                    }

                };

                let path = parsed.path();

                let url = format!("https://{}/repos/{}/git/trees/master?recursive=1", self.domain(), path);

                let url = format!("https://api.{}/repos{}/git/trees/main?recursive=1", self.domain(), path);

                debug!(url = url.as_str(), "=> {}", Label::using());

                Some(url)

            }
@@ -352,7 +353,7 @@ impl Bucket { 
            let folder = format!("{}/{}", output.display(), parent(path.clone()));

            std::fs::create_dir_all(folder.clone()).unwrap();

            if let Ok(mut file) = File::create(format!("{}/{}", output.display(), path)) {

                match client.get(url).send() {

                match client.get(url).header(USER_AGENT, "rust-web-api-client").send() {

                    | Ok(response) => match response.bytes() {

                        | Ok(bytes) => {

                            let mut content = Cursor::new(bytes);
@@ -391,15 +392,19 @@ impl Bucket { 
            parse_header(headers, "x-total-pages")

        }

        match self.tree(directory, FIRST_PAGE) {

            | Ok(response) if response.status().is_success() => (FIRST_PAGE..=page_count(&response))

            | Ok(response) if response.status().is_success() => {

                let paths = (FIRST_PAGE..=page_count(&response))

                    .into_par_iter()

                    .map(|page| self.clone().file_paths_for_page(directory, page))

                .reduce(std::vec::Vec::new, |a, b| [a, b].concat()),

                    .reduce(std::vec::Vec::new, |a, b| [a, b].concat());

                println!("{:#?}", response);

                paths

            },

            | Ok(_) | Err(_) => {

                let url = self.tree_url(directory, FIRST_PAGE);

                debug!(url, "=> {}", Label::using());

                error!(

                    "=> {} Failed to get file paths for {} bucket",

                    "=> {} Get file paths for {} bucket",

                    Label::fail(),

                    self.name.to_uppercase().red(),

                );

tests/fixtures/buckets_with_github.json

+0 −16
Original line number Diff line number Diff line 
{

    "buckets": [

        {

            "name": "nssd",

            "repository": {

                "provider": "gitlab",

                "id": 17410,

                "uri": "https://code.ornl.gov/research-enablement/buckets/nssd"

            }

        },

        {

            "name": "ccsd",

            "repository": {
@@ -16,14 +8,6 @@ 
                "uri": "https://code.ornl.gov/research-enablement/buckets/ccsd"

            }

        },

        {

            "name": "bessd",

            "repository": {

                "provider": "gitlab",

                "id": 17410,

                "uri": "https://code.ornl.gov/research-enablement/buckets/bessd"

            }

        },

        {

            "name": "test",

            "repository": {