feat: Add full support for GitHub buckets (7365c738) · Commits · Research Enablement / ACORN

acorn-lib/src/lib.rs

+86 −41

Original line number	Diff line number	Diff line
		@@ -172,6 +172,7 @@ pub struct GithubTreeEntry {
		///
		/// [GitHub]: https://docs.github.com/en/rest
		/// [documentation]: https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
		#[derive(Clone, Debug, Serialize, Deserialize)]
		pub struct GithubTreeResponse {
		/// SHA1 of tree
		pub sha: String,
		@@ -222,15 +223,30 @@ pub struct Release {
		pub released_at: String,
		}
		impl Bucket {
		/// Parse GitHub tree entries
		fn parse_github_response(response: reqwest::blocking::Response) -> Vec<String> {
		let content = response.text().unwrap();
		let data: Result<GithubTreeResponse> = serde_json::from_str(&content);
		match data {
		\| Ok(GithubTreeResponse { tree, .. }) => {
		debug!("=> {} {} Tree entries", Label::found(), tree.len());
		tree.into_iter().filter(GithubTreeEntry::is_blob).map(GithubTreeEntry::path).collect()
		}
		\| Err(why) => {
		error!("=> {} Process tree entries - {why}", Label::fail());
		vec![]
		}
		}
		}
		/// Parse GitLab tree entries
		fn parse(response: reqwest::blocking::Response) -> Vec<String> {
		fn parse_gitlab_response(response: reqwest::blocking::Response) -> Vec<String> {
		let content = response.text().unwrap();
		let data: Result<Vec<GitlabTreeEntry>> = serde_json::from_str(&content);
		debug!("=> {} {} GitLab tree entries", Label::found(), data.as_ref().unwrap().len());
		debug!("=> {} {} Tree entries", Label::found(), data.as_ref().unwrap().len());
		match data {
		\| Ok(entries) => entries.into_iter().filter(GitlabTreeEntry::is_blob).map(GitlabTreeEntry::path).collect(),
		\| Err(_) => {
		error!("=> {} Failed to process GitLab tree entries", Label::fail());
		\| Err(why) => {
		error!("=> {} Process tree entries - {why}", Label::fail());
		vec![]
		}
		}
		@@ -248,12 +264,12 @@ impl Bucket {
		},
		}
		}
		fn tree(&self, directory: &str, page: u32) -> eyre::Result<reqwest::blocking::Response, reqwest::Error> {
		fn tree(&self, directory: &str, page: Option<u32>) -> eyre::Result<reqwest::blocking::Response, reqwest::Error> {
		let url = self.tree_url(directory, page);
		let client = Client::new();
		client.get(url.unwrap_or_default()).header(USER_AGENT, "rust-web-api-client").send()
		}
		fn tree_url(&self, directory: &str, page: u32) -> Option<String> {
		fn tree_url(&self, directory: &str, page: Option<u32>) -> Option<String> {
		match &self.code_repository {
		\| Repository::GitHub { uri } => {
		let parsed = match URI::try_from(uri.as_str()) {
		@@ -276,7 +292,7 @@ impl Bucket {
		self.domain(),
		id,
		per_page,
		page,
		page.unwrap_or_default(),
		directory
		);
		debug!(url = url.as_str(), "=> {}", Label::using());
		@@ -344,15 +360,12 @@ impl Bucket {
		let progress = ProgressBar::new(paths.len() as u64);
		let client = Client::new();
		paths.par_iter().for_each(\|path\| {
		let url = match &self.code_repository {
		\| Repository::GitHub { uri, .. } => format!("{uri}/refs/heads/main/{path}"),
		\| Repository::GitLab { uri, .. } => format!("{uri}/-/raw/main/{path}"),
		};
		progress.set_style(ProgressStyle::with_template(Label::PROGRESS_BAR_TEMPLATE).unwrap());
		progress.set_message(format!("Downloading {path}"));
		let folder = format!("{}/{}", output.display(), parent(path.clone()));
		std::fs::create_dir_all(folder.clone()).unwrap();
		if let Ok(mut file) = File::create(format!("{}/{}", output.display(), path)) {
		if let Some(url) = self.code_repository.raw_url(path.to_string()) {
		match client.get(url).header(USER_AGENT, "rust-web-api-client").send() {
		\| Ok(response) => match response.bytes() {
		\| Ok(bytes) => {
		@@ -360,11 +373,12 @@ impl Bucket {
		let _ = copy(&mut content, &mut file);
		}
		\| Err(why) => {
		error!(path, "=> {} Failed to convert to bytes - {why}", Label::fail());
		error!(path, "=> {} Convert to bytes - {why}", Label::fail());
		}
		},
		\| Err(why) => {
		error!(path, "=> {} Failed to download - {why}", Label::fail());
		error!(path, "=> {} Download file - {why}", Label::fail());
		}
		}
		}
		};
		@@ -374,9 +388,8 @@ impl Bucket {
		progress.finish_with_message(message);
		total_data + total_images
		}
		// TODO: Verify pagination works the same for GitHub
		fn file_paths(self: Bucket, directory: &str) -> Vec<String> {
		const FIRST_PAGE: u32 = 1;
		const FIRST_PAGE: Option<u32> = Some(1);
		fn page_count(response: &reqwest::blocking::Response) -> u32 {
		fn parse_header(headers: &HeaderMap, key: &str) -> u32 {
		match headers.get(key) {
		@@ -391,11 +404,21 @@ impl Bucket {
		let headers = response.headers();
		parse_header(headers, "x-total-pages")
		}
		match self.tree(directory, FIRST_PAGE) {
		match self.code_repository {
		\| Repository::GitHub { .. } => match self.tree(directory, None) {
		\| Ok(response) if response.status().is_success() => Bucket::parse_github_response(response),
		\| Ok(_) \| Err(_) => {
		let url = self.tree_url(directory, None);
		debug!(url, "=> {}", Label::using());
		error!("=> {} Get file paths for {} bucket", Label::fail(), self.name.to_uppercase().red());
		vec![]
		}
		},
		\| Repository::GitLab { .. } => match self.tree(directory, FIRST_PAGE) {
		\| Ok(response) if response.status().is_success() => {
		let paths = (FIRST_PAGE..=page_count(&response))
		let paths = (FIRST_PAGE.unwrap_or_default()..=page_count(&response))
		.into_par_iter()
		.map(\|page\| self.clone().file_paths_for_page(directory, page))
		.map(\|page\| self.clone().file_paths_for_page(directory, Some(page)))
		.reduce(std::vec::Vec::new, \|a, b\| [a, b].concat());
		trace!("{:#?}", response);
		paths
		@@ -403,17 +426,18 @@ impl Bucket {
		\| Ok(_) \| Err(_) => {
		let url = self.tree_url(directory, FIRST_PAGE);
		debug!(url, "=> {}", Label::using());
		error!("=> {} Get file paths for {} bucket", Label::fail(), self.name.to_uppercase().red(),);
		error!("=> {} Get file paths for {} bucket", Label::fail(), self.name.to_uppercase().red());
		vec![]
		}
		},
		}
		}
		fn file_paths_for_page(self: Bucket, directory: &str, page: u32) -> Vec<String> {
		fn file_paths_for_page(self: Bucket, directory: &str, page: Option<u32>) -> Vec<String> {
		match self.tree(directory, page) {
		\| Ok(response) if response.status().is_success() => match self.tree(directory, page) {
		\| Ok(response) if response.status().is_success() => Bucket::parse(response),
		\| Ok(response) if response.status().is_success() => Bucket::parse_gitlab_response(response),
		\| Ok(_) \| Err(_) => {
		let url = self.tree_url(directory, 1);
		let url = self.tree_url(directory, Some(1));
		error!(url, page, "=> {} Failed to get paths", Label::fail());
		vec![]
		}
		@@ -449,6 +473,14 @@ impl BucketsConfig {
		data
		}
		}
		impl GithubTreeEntry {
		fn path(self) -> String {
		self.path
		}
		fn is_blob(&self) -> bool {
		self.entry_type.eq(&EntryType::Blob)
		}
		}
		impl GitlabTreeEntry {
		fn path(self) -> String {
		self.path
		@@ -537,6 +569,19 @@ impl Repository {
		vec![]
		}
		}
		/// Get URL for raw data of a file at a given path
		fn raw_url(&self, path: String) -> Option<String> {
		match self {
		\| Repository::GitHub { uri, .. } => match URI::try_from(uri.clone().as_str()) {
		\| Ok(ref value) => Some(format!("https://raw.githubusercontent.com{}/refs/heads/main/{path}", value.path())),
		\| Err(why) => {
		error!(uri, "=> {} Parse GitHub URI - {why}", Label::fail());
		None
		}
		},
		\| Repository::GitLab { uri, .. } => Some(format!("{uri}/-/raw/main/{path}")),
		}
		}
		}

		#[cfg(test)]

lcov.info

+177 −126

Original line number	Diff line number	Diff line
		@@ -1088,146 +1088,164 @@ LH:110
		end_of_record
		TN:
		SF:C:\Users\jason\dev\acorn\acorn-lib\src\lib.rs
		FN:226,Bucket::parse
		FN:239,Bucket::domain
		FN:251,Bucket::tree
		FN:255,Bucket::tree_url
		FN:292,Bucket::download_files
		FN:369,Bucket::file_paths
		FN:374,Bucket::file_paths::page_count
		FN:375,Bucket::file_paths::page_count::parse_header
		FN:407,Bucket::file_paths_for_page
		FN:427,BucketsConfig::read_json
		FN:449,GitlabTreeEntry::path
		FN:452,GitlabTreeEntry::is_blob
		FN:458,Repository::latest_release
		FN:468,Repository::id
		FN:489,Repository::releases
		FNF:15
		FNDA:0,Bucket::parse
		FN:227,Bucket::parse_github_response
		FN:242,Bucket::parse_gitlab_response
		FN:255,Bucket::domain
		FN:267,Bucket::tree
		FN:272,Bucket::tree_url
		FN:309,Bucket::download_files
		FN:310,Bucket::download_files::count_json_files
		FN:313,Bucket::download_files::count_image_files
		FN:316,Bucket::download_files::download_complete_message
		FN:348,Bucket::download_files::has_image_extension
		FN:391,Bucket::file_paths
		FN:393,Bucket::file_paths::page_count
		FN:394,Bucket::file_paths::page_count::parse_header
		FN:435,Bucket::file_paths_for_page
		FN:455,BucketsConfig::read_json
		FN:477,GithubTreeEntry::path
		FN:480,GithubTreeEntry::is_blob
		FN:485,GitlabTreeEntry::path
		FN:488,GitlabTreeEntry::is_blob
		FN:494,Repository::latest_release
		FN:504,Repository::id
		FN:525,Repository::releases
		FN:573,Repository::raw_url
		FNF:23
		FNDA:0,Bucket::parse_github_response
		FNDA:0,Bucket::parse_gitlab_response
		FNDA:144115188075855872,Bucket::domain
		FNDA:0,Bucket::tree
		FNDA:0,Bucket::tree_url
		FNDA:0,Bucket::download_files
		FNDA:0,Bucket::download_files::count_json_files
		FNDA:0,Bucket::download_files::count_image_files
		FNDA:0,Bucket::download_files::download_complete_message
		FNDA:0,Bucket::download_files::has_image_extension
		FNDA:0,Bucket::file_paths
		FNDA:0,Bucket::file_paths::page_count
		FNDA:0,Bucket::file_paths::page_count::parse_header
		FNDA:0,Bucket::file_paths_for_page
		FNDA:72057594037927936,BucketsConfig::read_json
		FNDA:0,GithubTreeEntry::path
		FNDA:0,GithubTreeEntry::is_blob
		FNDA:72057594037927936,GitlabTreeEntry::path
		FNDA:72057594037927936,GitlabTreeEntry::is_blob
		FNDA:144115188075855872,Repository::latest_release
		FNDA:72057594037927936,Repository::id
		FNDA:144115188075855872,Repository::releases
		DA:226,0
		FNDA:0,Repository::raw_url
		DA:227,0
		DA:228,0
		DA:229,0
		DA:230,0
		DA:231,0
		DA:232,0
		DA:233,0
		DA:234,0
		DA:239,144115188075855872
		DA:240,144115188075855872
		DA:241,72057594037927936
		DA:242,72057594037927936
		DA:235,0
		DA:236,0
		DA:237,0
		DA:242,0
		DA:243,0
		DA:245,72057594037927936
		DA:246,72057594037927936
		DA:244,0
		DA:245,0
		DA:246,0
		DA:247,0
		DA:251,0
		DA:252,0
		DA:253,0
		DA:255,0
		DA:256,0
		DA:257,0
		DA:258,0
		DA:248,0
		DA:249,0
		DA:250,0
		DA:255,144115188075855872
		DA:256,144115188075855872
		DA:257,72057594037927936
		DA:258,72057594037927936
		DA:259,0
		DA:260,0
		DA:261,0
		DA:262,0
		DA:265,0
		DA:266,0
		DA:261,72057594037927936
		DA:262,72057594037927936
		DA:263,0
		DA:267,0
		DA:268,0
		DA:271,0
		DA:269,0
		DA:270,0
		DA:272,0
		DA:273,0
		DA:274,0
		DA:275,0
		DA:276,0
		DA:277,0
		DA:278,0
		DA:279,0
		DA:281,0
		DA:282,0
		DA:283,0
		DA:284,0
		DA:285,0
		DA:288,0
		DA:289,0
		DA:290,0
		DA:292,0
		DA:293,0
		DA:294,0
		DA:295,0
		DA:296,0
		DA:297,0
		DA:298,0
		DA:299,0
		DA:301,0
		DA:303,0
		DA:304,0
		DA:305,0
		DA:306,0
		DA:307,0
		DA:308,0
		DA:309,0
		DA:310,0
		DA:311,0
		DA:312,0
		DA:313,0
		DA:314,0
		DA:315,0
		DA:316,0
		DA:317,0
		DA:318,0
		DA:319,0
		DA:320,0
		DA:322,0
		DA:323,0
		DA:324,0
		DA:326,0
		DA:327,0
		DA:328,0
		DA:329,0
		DA:330,0
		DA:332,0
		DA:334,0
		DA:333,0
		DA:335,0
		DA:336,0
		DA:337,0
		DA:338,0
		DA:339,0
		DA:341,0
		DA:343,0
		DA:344,0
		DA:345,0
		DA:346,0
		DA:347,0
		DA:350,0
		DA:352,0
		DA:353,0
		DA:354,0
		DA:348,0
		DA:349,0
		DA:351,0
		DA:355,0
		DA:356,0
		DA:357,0
		DA:358,0
		DA:359,0
		DA:360,0
		DA:361,0
		DA:362,0
		DA:363,0
		DA:364,0
		DA:365,0
		DA:366,0
		DA:367,0
		DA:368,0
		DA:369,0
		DA:370,0
		DA:371,0
		DA:374,0
		DA:372,0
		DA:373,0
		DA:375,0
		DA:376,0
		DA:377,0
		DA:378,0
		DA:379,0
		DA:380,0
		DA:382,0
		DA:385,0
		DA:386,0
		DA:387,0
		DA:388,0
		DA:389,0
		DA:391,0
		DA:392,0
		DA:393,0
		DA:394,0
		DA:395,0
		DA:396,0
		@@ -1235,80 +1253,113 @@ DA:397,0
		DA:398,0
		DA:399,0
		DA:401,0
		DA:404,0
		DA:405,0
		DA:407,0
		DA:408,0
		DA:409,0
		DA:410,0
		DA:411,0
		DA:412,0
		DA:413,0
		DA:414,0
		DA:417,0
		DA:418,0
		DA:419,0
		DA:420,0
		DA:427,72057594037927936
		DA:428,144115188075855872
		DA:429,144115188075855872
		DA:431,0
		DA:432,0
		DA:433,0
		DA:434,0
		DA:439,72057594037927936
		DA:440,144115188075855872
		DA:441,72057594037927936
		DA:421,0
		DA:422,0
		DA:423,0
		DA:424,0
		DA:427,0
		DA:428,0
		DA:429,0
		DA:430,0
		DA:435,0
		DA:436,0
		DA:437,0
		DA:438,0
		DA:440,0
		DA:441,0
		DA:442,0
		DA:444,72057594037927936
		DA:445,72057594037927936
		DA:449,72057594037927936
		DA:450,72057594037927936
		DA:452,72057594037927936
		DA:453,72057594037927936
		DA:458,144115188075855872
		DA:459,144115188075855872
		DA:460,144115188075855872
		DA:461,144115188075855872
		DA:462,144115188075855872
		DA:463,144115188075855872
		DA:468,72057594037927936
		DA:446,0
		DA:447,0
		DA:448,0
		DA:455,72057594037927936
		DA:456,144115188075855872
		DA:457,144115188075855872
		DA:459,0
		DA:460,0
		DA:461,0
		DA:462,0
		DA:467,72057594037927936
		DA:468,144115188075855872
		DA:469,72057594037927936
		DA:471,72057594037927936
		DA:470,0
		DA:472,72057594037927936
		DA:473,72057594037927936
		DA:474,72057594037927936
		DA:475,72057594037927936
		DA:476,72057594037927936
		DA:479,0
		DA:477,0
		DA:478,0
		DA:480,0
		DA:481,0
		DA:482,0
		DA:483,0
		DA:489,144115188075855872
		DA:490,288230376151711744
		DA:491,72057594037927936
		DA:492,72057594037927936
		DA:493,72057594037927936
		DA:494,72057594037927936
		DA:495,72057594037927936
		DA:498,0
		DA:499,0
		DA:502,72057594037927936
		DA:503,72057594037927936
		DA:504,0
		DA:507,288230376151711744
		DA:508,0
		DA:511,144115188075855872
		DA:512,144115188075855872
		DA:514,144115188075855872
		DA:485,72057594037927936
		DA:486,72057594037927936
		DA:488,72057594037927936
		DA:489,72057594037927936
		DA:494,144115188075855872
		DA:495,144115188075855872
		DA:496,144115188075855872
		DA:497,144115188075855872
		DA:498,144115188075855872
		DA:499,144115188075855872
		DA:504,72057594037927936
		DA:505,72057594037927936
		DA:506,0
		DA:507,72057594037927936
		DA:508,72057594037927936
		DA:509,72057594037927936
		DA:510,72057594037927936
		DA:511,72057594037927936
		DA:512,72057594037927936
		DA:515,0
		DA:516,0
		DA:517,0
		DA:522,0
		DA:523,0
		DA:524,0
		DA:527,0
		DA:528,0
		DA:529,0
		DA:533,0
		LF:189
		DA:518,0
		DA:519,0
		DA:525,144115188075855872
		DA:526,288230376151711744
		DA:527,72057594037927936
		DA:528,72057594037927936
		DA:529,72057594037927936
		DA:530,72057594037927936
		DA:531,72057594037927936
		DA:534,0
		DA:535,0
		DA:538,72057594037927936
		DA:539,72057594037927936
		DA:540,0
		DA:543,288230376151711744
		DA:544,0
		DA:547,144115188075855872
		DA:548,144115188075855872
		DA:550,144115188075855872
		DA:551,0
		DA:552,0
		DA:553,0
		DA:558,0
		DA:559,0
		DA:560,0
		DA:563,0
		DA:564,0
		DA:565,0
		DA:569,0
		DA:573,0
		DA:574,0
		DA:575,0
		DA:576,0
		DA:577,0
		DA:578,0
		DA:579,0
		DA:582,0
		LF:224
		LH:45
		end_of_record
		TN:
		@@ -2277,7 +2328,7 @@ FNDA:144115188075855872,checksum
		FNDA:2089670227099910144,command_exists
		FNDA:0,download_binary
		FNDA:0,download_binary::download
		FNDA:1080863910568919040,extension
		FNDA:648518346341351424,extension
		FNDA:216172782113783808,files_all
		FNDA:216172782113783808,files_all::paths_to_vec
		FNDA:72057594037927936,files_from_git_branch
		@@ -2444,8 +2495,8 @@ DA:475,0
		DA:476,0
		DA:477,0
		DA:478,0
		DA:492,1080863910568919040
		DA:493,1080863910568919040
		DA:492,648518346341351424
		DA:493,648518346341351424
		DA:506,216172782113783808
		DA:507,216172782113783808
		DA:508,2738188573441261568
		@@ -2498,7 +2549,7 @@ DA:593,144115188075855872
		DA:594,144115188075855872
		DA:595,144115188075855872
		DA:598,144115188075855872
		DA:599,936748722493063168
		DA:599,504403158265495552
		DA:601,0
		DA:603,0
		DA:615,144115188075855872