feat: Add gitlab programming languages as persistent application data (52a5402d) · Commits · Research Enablement / ACORN

acorn-cli/src/commands/gather/mod.rs

+3 −1

Original line number	Diff line number	Diff line
		@@ -28,7 +28,9 @@ pub async fn run(
		println!("CiteAs API is healthy: {}", api::citeas::is_healthy().await);
		println!("ORCiD API is healthy: {}", api::orcid::is_healthy().await);
		println!("ROR API is healthy: {}", api::ror::is_healthy().await);
		gitlab_merge_request_note_example().await;
		// let langs = gitlab::languages().await?;
		// dbg!(langs);
		// gitlab_merge_request_note_example().await;
		// gitlab_example().await;
		// orcid_example().await;
		// ror_example().await;

acorn-cli/src/main.rs

+1 −0

Original line number	Diff line number	Diff line
		@@ -198,6 +198,7 @@ async fn main() -> Void {
		database.with_connection(\|_\| Ok(())).map(\|_\| ())
		}))
		.and_then(\|_\| database.populate(Table::Licenses))
		.and_then(\|_\| database.populate(Table::ProgrammingLanguages))
		.await
		.map(\|_\| ())
		};

acorn-lib/assets/constants/application.json

+22 −0

Original line number	Diff line number	Diff line
		@@ -100,6 +100,28 @@
		}
		]
		},
		{
		"name": "languages::gitlab",
		"domain": "gitlab.com",
		"resources": [
		{
		"name": "languages",
		"method": "get",
		"template": "{{ base }}/gitlab-org/gitlab/-/raw/master/vendor/languages.yml"
		}
		]
		},
		{
		"name": "languages::github",
		"domain": "raw.githubusercontent.com",
		"resources": [
		{
		"name": "languages",
		"method": "get",
		"template": "{{ base }}/github-linguist/linguist/refs/heads/main/lib/linguist/languages.yml"
		}
		]
		},
		{
		"name": "orcid",
		"domain": "pub.orcid.org",

acorn-lib/src/io/api/gitlab.rs

+159 −3

Original line number	Diff line number	Diff line
		@@ -2,12 +2,18 @@
		//!
		// TODO: Add custom field list and query pair type
		// TODO: Finish modeling necessary API endpoints
		use crate::io::api::{EmptyField, Endpoint, RemoteResource, Resource, TreeEntryType, ValueValidator, INCLUDED_ENDPOINTS};
		use crate::io::api::{
		DatabasePersistence, EmptyField, Endpoint, RemoteResource, Resource, ResponseContent, TreeEntryType, ValueValidator, INCLUDED_ENDPOINTS,
		};
		use crate::io::config::{RunnerStatus, RunnerType};
		use crate::io::ApiResult;
		use crate::io::database::schema::{ProgrammingLanguageRow, Table};
		use crate::io::database::{Database, Operations};
		use crate::io::{with_progress, ApiResult, ProgressType};
		use crate::param;
		use crate::prelude::var;
		use crate::util::Searchable;
		use crate::prelude::HashMap;
		use crate::util::{Label, Searchable};
		use async_trait::async_trait;
		use bon::Builder;
		use color_eyre::eyre::{self, eyre};
		use core::fmt;
		@@ -21,6 +27,8 @@ pub type GroupsResponse = Vec<GroupDetails>;
		pub type RunnersResponse = Vec<RunnerDetails>;
		/// Type for GitLab API response for a tree
		pub type TreeResponse = Vec<TreeEntry>;
		/// Type for GitLab programming language entries
		pub type ProgrammingLanguageEntries = Vec<ProgrammingLanguageMetadata>;
		/// Access level of the runner
		#[derive(Clone, Debug, Serialize, Deserialize)]
		#[serde(rename_all = "snake_case")]
		@@ -30,6 +38,41 @@ pub enum AccessLevel {
		/// Ref protected
		RefProtected,
		}
		/// Language metadata details from GitLab languages YAML file
		#[skip_serializing_none]
		#[derive(Clone, Debug, Default, Serialize, Deserialize)]
		pub struct ProgrammingLanguageDetails {
		/// Canonical language identifier
		pub language_id: Option<u64>,
		/// Language category (for example, `programming`, `data`, `markup`, or `prose`)
		#[serde(rename = "type")]
		pub language_type: Option<String>,
		/// Display color (hex string)
		pub color: Option<String>,
		/// Optional parent language group
		pub group: Option<String>,
		}
		/// Normalized programming language metadata with explicit language name
		#[skip_serializing_none]
		#[derive(Clone, Debug, Default, Serialize, Deserialize)]
		pub struct ProgrammingLanguageMetadata {
		/// Display name of the language
		pub name: String,
		/// Canonical language identifier
		pub language_id: Option<u64>,
		/// Language category (for example, `programming`, `data`, `markup`, or `prose`)
		pub language_type: Option<String>,
		/// Display color (hex string)
		pub color: Option<String>,
		/// Optional parent language group
		pub group: Option<String>,
		}
		/// Parsed response for GitLab language metadata
		#[derive(Clone, Debug, Default, Serialize)]
		pub struct ProgrammingLanguagesResponse {
		/// Flattened language metadata entries
		pub languages: ProgrammingLanguageEntries,
		}
		/// Group visibility level
		#[derive(Clone, Debug, Default, Serialize, Deserialize)]
		#[serde(rename_all = "snake_case")]
		@@ -558,6 +601,66 @@ impl Options {
		}
		}
		}
		impl From<ProgrammingLanguageMetadata> for ProgrammingLanguageRow {
		fn from(value: ProgrammingLanguageMetadata) -> Self {
		let ProgrammingLanguageMetadata {
		name,
		language_id,
		language_type,
		color,
		group,
		} = value;
		ProgrammingLanguageRow::init()
		.name(name)
		.maybe_language_id(language_id.and_then(\|value\| i64::try_from(value).ok()))
		.maybe_language_type(language_type)
		.maybe_color(color)
		.maybe_group_name(group)
		.build()
		}
		}
		impl ProgrammingLanguagesResponse {
		/// Parse a raw language map, retaining only `programming` type entries
		pub fn parse(data: HashMap<String, ProgrammingLanguageDetails>) -> Self {
		let languages = data
		.into_iter()
		.filter_map(\|(name, details)\| {
		details
		.language_type
		.as_ref()
		.map(\|kind\| kind.eq_ignore_ascii_case("programming"))
		.filter(\|is_programming\| *is_programming)
		.map(\|_\| ProgrammingLanguageMetadata {
		name,
		language_id: details.language_id,
		language_type: details.language_type,
		color: details.color,
		group: details.group,
		})
		})
		.collect();
		Self { languages }
		}
		}
		impl<'de> serde::Deserialize<'de> for ProgrammingLanguagesResponse {
		fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
		HashMap::<String, ProgrammingLanguageDetails>::deserialize(deserializer).map(Self::parse)
		}
		}
		#[async_trait]
		impl DatabasePersistence for ProgrammingLanguagesResponse {
		/// Persist GitLab programming language metadata to local database
		async fn persist(self, database: Database<Table>) -> ApiResult<usize> {
		let Self { languages } = self;
		let message: fn(&ProgrammingLanguageMetadata) -> String = \|item\| format!("Saving \"{}\" language metadata", item.name);
		let operation = \|item\| async { database.insert(ProgrammingLanguageRow::from(item)) };
		let finish = \|count\| format!("{}Saved metadata for {count} programming languages", Label::CHECKMARK);
		with_progress(languages, message, operation, finish, None, ProgressType::Bar)
		.await
		.map(\|counts\| counts.into_iter().sum())
		.map_err(eyre::Report::msg)
		}
		}
		impl Default for Options {
		fn default() -> Self {
		Self::from_env()
		@@ -667,6 +770,31 @@ pub async fn groups(options: Options) -> ApiResult<GroupsResponse> {
		\| None => Err(eyre!("{name} API endpoint not found")),
		}
		}
		/// Download programming language metadata from GitLab linguist source file
		pub async fn languages() -> ApiResult<ProgrammingLanguagesResponse> {
		let names = ["languages::gitlab", "languages::github"];
		let action = "languages";
		async fn fetch(name: &str, action: &str) -> ApiResult<ProgrammingLanguagesResponse> {
		match INCLUDED_ENDPOINTS.find_by_name(name) {
		\| Some(endpoint) => {
		let response = endpoint.invoke(action, None).await.map(\|content\| match content {
		\| ResponseContent::Raw(content) => ResponseContent::Yaml(content),
		\| other => other,
		});
		endpoint.handle::<ProgrammingLanguagesResponse>(response)
		}
		\| None => Err(eyre!("{name} API endpoint not found")),
		}
		}
		let mut errors = Vec::new();
		for name in names {
		match fetch(name, action).await {
		\| Ok(response) => return Ok(response),
		\| Err(why) => errors.push(format!("{name}={why}")),
		}
		}
		Err(eyre!("Failed to download and parse language metadata — {}", errors.join("; ")))
		}
		/// Add a note (comment) to a merge request using project identifier and merge request IID
		///
		/// When used in CI environment, the project identifier can be obtained from the `CI_PROJECT_ID` environment variable and the merge request IID can be obtained from the `CI_MERGE_REQUEST_IID` environment variable.
		@@ -759,6 +887,7 @@ pub(crate) async fn tree_paths(
		mod tests {
		use super::*;
		use crate::io::api::IntoBody;
		use core::iter::FromIterator;

		#[test]
		fn test_endpoint_for_base_url() {
		@@ -775,4 +904,31 @@ mod tests {
		let payload = vec![param!(Body, "body", "This is a test comment!!!")].into_body();
		assert_eq!(payload, serde_json::json!({ "body": "This is a test comment!!!" }));
		}
		#[test]
		fn test_programming_languages_response_parse_filters_programming_only() {
		let data = HashMap::from_iter([
		(
		"Python".to_string(),
		ProgrammingLanguageDetails {
		language_id: Some(303),
		language_type: Some("programming".to_string()),
		color: Some("#3572A5".to_string()),
		group: None,
		},
		),
		(
		"YAML".to_string(),
		ProgrammingLanguageDetails {
		language_id: Some(407),
		language_type: Some("data".to_string()),
		color: Some("#cb171e".to_string()),
		group: None,
		},
		),
		]);
		let response = ProgrammingLanguagesResponse::parse(data);
		assert_eq!(response.languages.len(), 1);
		assert_eq!(response.languages[0].name, "Python");
		assert_eq!(response.languages[0].language_id, Some(303));
		}
		}

acorn-lib/src/io/api/mod.rs

+12 −0

Original line number	Diff line number	Diff line
		@@ -134,6 +134,8 @@ pub enum ResponseContent {
		Json(String),
		/// Raw text response content
		Raw(String),
		/// YAML response content
		Yaml(String),
		/// XML response content
		Xml(String),
		}
		@@ -504,6 +506,7 @@ impl RemoteResource for Endpoint {
		\| Ok(content) => match content {
		\| ResponseContent::Json(content) => parse_json(&content),
		\| ResponseContent::Xml(content) => parse_xml(&content),
		\| ResponseContent::Yaml(content) => parse_yaml(&content),
		\| ResponseContent::Raw(content) => {
		let raw = TextResponse { content };
		serde_json::to_string(&raw).map_err(\|e\| eyre!(e)).and_then(\|json\| parse_json(&json))
		@@ -717,6 +720,15 @@ where
		\| Err(why) => Err(eyre!(why)),
		}
		}
		pub(crate) fn parse_yaml<R>(content: &str) -> ApiResult<R>
		where
		R: for<'de> Deserialize<'de>,
		{
		match serde_yml::from_str::<R>(content) {
		\| Ok(response) => Ok(response),
		\| Err(why) => Err(eyre!(why)),
		}
		}
		/// Construct a query string for an endpoint API query from a list of field-value pairs, a list of fields, and a list of fields with boosted relevance.
		///
		/// The query string is constructed by joining the following parts with "&":