Commit 71e5bdae authored by Wohlgemuth, Jason's avatar Wohlgemuth, Jason
Browse files

feat: More remote resource API refinement

parent 1d6d9b99

acorn-cli/src/commands/gather/mod.rs

+4 −16
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ pub async fn run(_path: &Option<PathBuf>, _ignore: &Option<String>, _offline: &b 
    println!("ORCiD API is healthy: {}", api::orcid::is_healthy().await);

    println!("ROR API is healthy: {}", api::ror::is_healthy().await);

    // spdx_example(&INCLUDED_ENDPOINTS).await;

    orcid_example().await;

    // orcid_example().await;

    ror_example().await;

    Ok(())

}
@@ -31,28 +31,16 @@ async fn orcid_example() { 
}

#[allow(dead_code)]

async fn ror_example() {

    // TODO: Unify record and search

    // ROR API examples

    // let ror = endpoints.find_by_name("ror");

    // let text = match &ror {

    //     | Some(endpoint) => {

    //         let data = vec![param!(TemplateValue, "identifier", "01qz5mb56")];

    //         let response = endpoint.invoke("record", Some(data)).await;

    //         endpoint.handle::<api::ror::SingleRecord>(response)

    //     }

    //     | None => Err("No ROR endpoint found".into()),

    // };

    // println!("ROR Record: {text:#?}");

    let identifier = "01qz5mb56";

    println!("ROR Record: {:#?}", api::ror::record(identifier).await);

    let params = vec![

        param!(FieldList, "query", "Oak Ridge"),

        param!(QueryPair, "filter", ("status", "inactive")),

    ];

    let text = api::ror::search(params).await;

    println!("ROR Search Results: {text:#?}");

    println!("ROR Search Results: {:#?}", api::ror::search(params).await);

}

#[allow(dead_code)]

async fn spdx_example(endpoints: &Vec<Endpoint>) {

    // Get SPDX license data

    let text = match endpoints.find_by_name("spdx") {

        | Some(endpoint) => {

            let response = endpoint.invoke("licenses", None).await;

acorn-lib/src/io/api/mod.rs

+71 −41
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ use lazy_static::lazy_static; 
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};

use serde::{Deserialize, Serialize};

use serde_with::skip_serializing_none;

use std::collections::HashSet;

use tera::{Context, Tera};

use uriparse::URI;

use validator::Validate;
@@ -76,14 +77,6 @@ pub trait RemoteResource { 
    where

        Q: QueryField + ValueValidator,

        F: QueryField;

    /// Parse JSON response string content

    fn parse_json<R>(&self, content: &str) -> Result<R, String>

    where

        R: for<'de> Deserialize<'de>;

    /// Parse XML response string content

    fn parse_xml<R>(&self, content: &str) -> Result<R, String>

    where

        R: for<'de> Deserialize<'de>;

}

/// Helper trait for searching lists of named API elements

pub trait Searchable<T> {
@@ -533,13 +526,11 @@ impl RemoteResource for Endpoint { 
    {

        match response {

            | Ok(content) => match content {

                | ResponseContent::Json(content) => self.parse_json(&content),

                | ResponseContent::Xml(content) => self.parse_xml(&content),

                | ResponseContent::Json(content) => parse_json(&content),

                | ResponseContent::Xml(content) => parse_xml(&content),

                | ResponseContent::Raw(content) => {

                    let raw = TextResponse { content };

                    serde_json::to_string(&raw)

                        .map_err(|e| e.to_string())

                        .and_then(|json| self.parse_json(&json))

                    serde_json::to_string(&raw).map_err(|e| e.to_string()).and_then(|json| parse_json(&json))

                }

            },

            | Err(e) => Err(e.to_string()),
@@ -590,15 +581,15 @@ impl RemoteResource for Endpoint { 
        F: QueryField,

    {

        let Self { resources, .. } = self;

        let mut tera = Tera::default();

        let context = self.context_with::<Q, F>(data.clone());

        let mut context = self.context_with::<Q, F>(data.clone());

        let resource = resources.find_by_name(name);

        match resource {

            | Some(Resource { method, template, .. }) => {

                let path = tera.render_str(&template, &context).unwrap_or_default();

                let path = render(&template, &mut context);

                let params = data.unwrap_or_default();

                let headers = params.clone().into_headers();

                let body = params.into_body();

                dbg!(&path);

                let request = match method {

                    | HttpMethod::Get => get(path),

                    | HttpMethod::Post => post(path).json(&body),
@@ -625,24 +616,6 @@ impl RemoteResource for Endpoint { 
            | None => Err(Error::new(ErrorKind::NotFound, "Resource not found")),

        }

    }

    fn parse_json<R>(&self, content: &str) -> Result<R, String>

    where

        R: for<'de> Deserialize<'de>,

    {

        match serde_json::from_str::<R>(content) {

            | Ok(response) => Ok(response),

            | Err(why) => Err(why.to_string()),

        }

    }

    fn parse_xml<R>(&self, content: &str) -> Result<R, String>

    where

        R: for<'de> Deserialize<'de>,

    {

        match quick_xml::de::from_str::<R>(content) {

            | Ok(response) => Ok(response),

            | Err(why) => Err(why.to_string()),

        }

    }

}

impl TryFrom<&str> for EmptyField {

    type Error = String;
@@ -656,8 +629,47 @@ impl ValueValidator for EmptyField { 
        true

    }

}

pub(crate) fn extract_template_keys(template: &str) -> Vec<String> {

    fn extract_key(expression: &str) -> Option<String> {

        let trimmed = expression.trim().trim_matches('-');

        let mut base_parts = trimmed.split('|');

        let base = match base_parts.next() {

            | Some(value) => value.trim(),

            | None => return None,

        };

        let mut key_parts = base.split_whitespace();

        let key = match key_parts.next() {

            | Some(value) => value.trim(),

            | None => return None,

        };

        if key.is_empty() {

            None

        } else {

            Some(key.to_string())

        }

    }

    let mut keys = Vec::new();

    let mut seen = HashSet::new();

    let mut remaining = template;

    while let Some(start) = remaining.find("{{") {

        remaining = &remaining[start + 2..];

        let end = match remaining.find("}}") {

            | Some(value) => value,

            | None => break,

        };

        let raw = &remaining[..end];

        if let Some(key) = extract_key(raw) {

            let key_value = key.to_string();

            if seen.insert(key_value.clone()) {

                keys.push(key_value);

            }

        }

        remaining = &remaining[end + 2..];

    }

    keys

}

/// Create a query string from a lookup table of key-value pairs with field validation

pub fn param_from_query_pairs<T: QueryField + ValueValidator>(key: &str, separator: &str, pairs: Vec<(&str, &str)>) -> Option<String> {

pub(crate) fn param_from_query_pairs<T: QueryField + ValueValidator>(key: &str, separator: &str, pairs: Vec<(&str, &str)>) -> Option<String> {

    let values: Vec<String> = pairs

        .into_iter()

        .filter_map(|(k, v)| {
@@ -682,7 +694,7 @@ pub fn param_from_query_pairs<T: QueryField + ValueValidator>(key: &str, separat 
    }

}

/// Create a query string from a list of field values

pub fn param_from_field_list<T: QueryField>(key: &str, separator: &str, fields: Vec<&str>) -> Option<String> {

pub(crate) fn param_from_field_list<T: QueryField>(key: &str, separator: &str, fields: Vec<&str>) -> Option<String> {

    let values: Vec<String> = fields

        .into_iter()

        .filter_map(|value: &str| {
@@ -700,7 +712,7 @@ pub fn param_from_field_list<T: QueryField>(key: &str, separator: &str, fields: 
    }

}

/// Create a boosted query string from a list of fields with weighted relevance

pub fn param_from_query_fields<T: QueryField>(key: &str, separator: &str, fields: Vec<&str>) -> Option<String> {

pub(crate) fn param_from_query_fields<T: QueryField>(key: &str, separator: &str, fields: Vec<&str>) -> Option<String> {

    let valid_fields: Vec<T> = fields.into_iter().filter_map(|value| T::try_from(value).ok()).collect();

    if valid_fields.is_empty() {

        None
@@ -718,14 +730,22 @@ pub fn param_from_query_fields<T: QueryField>(key: &str, separator: &str, fields 
        ))

    }

}

/// Parse API response content into a structured data type using `quick_xml` for XML deserialization

pub fn parse<R>(content: &str) -> Result<R, String>

pub(crate) fn parse_json<R>(content: &str) -> Result<R, String>

where

    R: for<'de> Deserialize<'de>,

{

    match serde_json::from_str::<R>(content) {

        | Ok(response) => Ok(response),

        | Err(why) => Err(why.to_string()),

    }

}

pub(crate) fn parse_xml<R>(content: &str) -> Result<R, String>

where

    R: for<'de> Deserialize<'de>,

{

    match quick_xml::de::from_str::<R>(content) {

        | Ok(response) => Ok(response),

        | Err(e) => Err(format!("Failed to parse ORCiD search response: {e}")),

        | Err(why) => Err(why.to_string()),

    }

}

/// Construct a query string for an endpoint API query from a list of field-value pairs, a list of fields, and a list of fields with boosted relevance.
@@ -737,7 +757,7 @@ where 
/// - The list of fields with boosted relevance, joined with URL encoded space, prefixed with "&qf=".

///

/// If the list of field-value pairs is empty, an empty string is returned.

pub fn query_string<Q: QueryField + ValueValidator, F: QueryField>(

pub(crate) fn query_string<Q: QueryField + ValueValidator, F: QueryField>(

    query_pairs: Vec<(&str, &str)>,

    field_list: Vec<&str>,

    query_fields: Vec<&str>,
@@ -749,6 +769,16 @@ pub fn query_string<Q: QueryField + ValueValidator, F: QueryField>( 
    ];

    Param::to_query_string::<Q, F>(params)

}

pub(crate) fn render(template: &str, context: &mut Context) -> String {

    let mut tera = Tera::default();

    let keys = extract_template_keys(template);

    keys.into_iter().for_each(|key| {

        if !context.contains_key(&key) {

            context.insert(&key, "");

        }

    });

    tera.render_str(template, context).unwrap_or_default()

}



#[cfg(test)]

mod tests;

acorn-lib/src/io/api/ror.rs

+31 −53
Original line number Diff line number Diff line
@@ -2,60 +2,9 @@ 
//!

//! Provides types and functions for constructing ROR API queries with field validation and output column selection

//!

//! ## Example Uses

//!

//! ### Get API status

//! ```ignore

//! use acorn_lib::io::api::{self, Search};

//!

//! let ror = endpoints.find_by_name("ror");

//! let text = match &ror {

//!     | Some(endpoint) => {

//!         let response = endpoint.invoke("status", None).await;

//!         endpoint.handle::<api::TextResponse>(response)

//!     }

//!     | None => Err("No ROR endpoint found".into()),

//! };

//! println!("ROR Status: {text:#?}");

//! ```

//!

//! ### Get single ROR record by ID

//! ```ignore

//! use acorn_lib::io::api::{ror, Search};

//! use acorn_lib::param;

//!

//! let ror = endpoints.find_by_name("ror");

//! let text = match &ror {

//!     | Some(endpoint) => {

//!         let data = vec![param!(TemplateValue, "identifier", "01qz5mb56")];

//!         let response = endpoint.invoke("record", Some(data)).await;

//!         endpoint.handle::<ror::SingleRecord>(response)

//!     }

//!     | None => Err("No ROR endpoint found".into()),

//! };

//! println!("ROR Record: {text:#?}");

//! ```

//!

//! ### Search and filter ROR records

//! ```ignore

//! use acorn_lib::io::api::{ror, Search};

//! use acorn_lib::param;

//!

//! let ror = endpoints.find_by_name("ror");

//! let text = match &ror {

//!     | Some(endpoint) => {

//!         let data = vec![

//!             param!(FieldList, "query", "Oak Ridge"),

//!             param!(QueryPair, "filter", (("status", "inactive"),)),

//!         ];

//!         let response = endpoint.invoke("search", Some(data)).await;

//!         endpoint.handle::<ror::SearchResponse>(response)

//!     }

//!     | None => Err("No ROR endpoint found".into()),

//! };

//! println!("ROR Search Results: {text:#?}");

//! ```

use crate::io::api::{Param, RemoteResource, Searchable, TextResponse, INCLUDED_ENDPOINTS};

use crate::param;

use crate::schema::pid::{PersistentIdentifierParse, ROR};

use bon::Builder;

use schemars::JsonSchema;

use serde::{Deserialize, Serialize};
@@ -406,6 +355,35 @@ pub async fn is_healthy() -> bool { 
        | Err(_) => false,

    }

}

/// Get a single ROR record by identifier

/// ### Note

/// Will only make a request if the identifier is valid ROR

///

/// ### Example

/// ```ignore

/// use acorn_lib::io::api::ror::{self, SingleRecord};

///

/// let identifier = "01qz5mb56";

/// let result: Result<SingleRecord, String> = ror::record(identifier).await;

/// ```

pub async fn record(identifier: impl ToString) -> Result<SingleRecord, String> {

    let value = identifier.to_string();

    if ROR::is_valid(&value) {

        let name = "ROR";

        let action = "search";

        let params = vec![param!(TemplateValue, "identifier", &value)];

        let data = Some(params);

        match INCLUDED_ENDPOINTS.find_by_name(name) {

            | Some(endpoint) => {

                let response = endpoint.invoke(action, data).await;

                endpoint.handle::<SingleRecord>(response)

            }

            | None => Err(format!("{name} API endpoint not found")),

        }

    } else {

        Err(format!("Invalid ROR identifier: {}", &value))

    }

}

/// Search the ROR API with given query parameters and output fields

///

/// ### Example

acorn-lib/src/io/api/tests/mod.rs

+30 −5
Original line number Diff line number Diff line 
use crate::io::api::citeas::{self, ToCitations};

use crate::io::api::{self, geonames, orcid, ror, Endpoint, Param, ParamStyle, RemoteResource, Resource, ResponseContent, Searchable};

use crate::io::api::{

    self, extract_template_keys, geonames, orcid, ror, Endpoint, Param, ParamStyle, RemoteResource, Resource, ResponseContent, Searchable,

};

use crate::io::read_file;

use crate::param;

use crate::schema::pid::{PersistentIdentifierParse, DOI};
@@ -137,15 +139,14 @@ fn test_endpoint_handle() { 
#[test]

fn test_endpoint_parse() {

    let xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n<expanded-search:expanded-search num-found=\"2\" xmlns:expanded-search=\"http://www.orcid.org/ns/expanded-search\">\n    <expanded-search:expanded-result>\n        <expanded-search:family-names>Carson</expanded-search:family-names>\n    </expanded-search:expanded-result>\n    <expanded-search:expanded-result>\n        <expanded-search:family-names>Wohlgemuth</expanded-search:family-names>\n    </expanded-search:expanded-result>\n</expanded-search:expanded-search>\n";

    let endpoint = Endpoint::at("pub.orcid.org").root("v3.0").build();

    let response = endpoint.parse_xml::<orcid::SearchResponse>(xml).expect("Failed to parse XML");

    let response = api::parse_xml::<orcid::SearchResponse>(xml).expect("Failed to parse XML");

    assert_eq!(response.num_found, 2);

    assert_eq!(response.namespace, "http://www.orcid.org/ns/expanded-search");

    assert_eq!(response.results.len(), 2);

    assert_eq!(response.results[0].family_names, Some("Carson".to_string()));

    assert_eq!(response.results[1].family_names, Some("Wohlgemuth".to_string()));

    let json = "{\"tomcatUp\":true,\"dbConnectionOk\":true,\"readOnlyDbConnectionOk\":false,\"overallOk\":true}";

    let status = endpoint.parse_json::<orcid::StatusResponse>(json).expect("Failed to parse JSON");

    let status = api::parse_json::<orcid::StatusResponse>(json).expect("Failed to parse JSON");

    assert!(status.application);

    assert!(status.database);

    assert!(!status.database_readonly);
@@ -273,11 +274,35 @@ fn test_orcid_query_string() { 
    assert_eq!(query, expected);

}

#[test]

fn test_extract_template_keys() {

    let template = "{{ base }}/organizations/{{ identifier }}{{ query }}";

    let keys = extract_template_keys(template);

    let expected = vec!["base", "identifier", "query"].into_iter().map(String::from).collect::<Vec<String>>();

    assert_eq!(keys, expected);

    let template = "{{ query | default(value=\"\") }} and {{- name -}}";

    let keys = extract_template_keys(template);

    let expected = vec!["query", "name"].into_iter().map(String::from).collect::<Vec<String>>();

    assert_eq!(keys, expected);

    let template = "{{ base }}/{{ base }}/{{ identifier }}";

    let keys = extract_template_keys(template);

    let expected = vec!["base", "identifier"].into_iter().map(String::from).collect::<Vec<String>>();

    assert_eq!(keys, expected);



    let template = "{{ base }}/{{ missing";

    let keys = extract_template_keys(template);

    let expected = vec!["base"].into_iter().map(String::from).collect::<Vec<String>>();

    assert_eq!(keys, expected);

    let template = "";

    let keys = extract_template_keys(template);

    let expected: Vec<String> = vec![];

    assert_eq!(keys, expected);

}

#[test]

fn test_orcid_response_handler() {

    // Load XML fixture file

    let xml = read_file(fixtures_dir().join("response_orcid_wohlgemuth.xml")).expect("Failed to read XML fixture");

    // Parse XML into OrcidSearchResponse

    let response = api::parse::<orcid::SearchResponse>(&xml).expect("Failed to deserialize XML");

    let response = api::parse_xml::<orcid::SearchResponse>(&xml).expect("Failed to deserialize XML");

    // Verify basic response properties

    let total = 68;

    assert_eq!(response.num_found, total);

acorn-lib/src/util/constants.rs

+1 −6
Original line number Diff line number Diff line
@@ -235,15 +235,10 @@ pub const ENDPOINTS_TEXT: &str = r#"{ 
            "domain": "api.ror.org",

            "root": "v2",

            "resources": [

                {

                    "name": "record",

                    "method": "get",

                    "template": "{{ base }}/organizations/{{ identifier }}"

                },

                {

                    "name": "search",

                    "method": "get",

                    "template": "{{ base }}/organizations/{{ query }}"

                    "template": "{{ base }}/organizations/{{ identifier }}{{ query }}"

                },

                {

                    "name": "status",