feat: Cull ModuleUri methods, retain only ModuleUri::from (c5569daf) · Commits · Research Enablement / Xylem

Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -18,7 +18,7 @@ lint:
		@cargo clippy --no-deps --all-targets --fix --allow-dirty

		test:
		@cargo nextest run --run-ignored all
		@cargo nextest run

		test-ci: clean lint
		@cargo test --lib --bins --tests

xylem-lib/src/config/mod.rs

+32 −30

Original line number	Diff line number	Diff line
		use acorn_lib::util::{download_binary, extension, make_executable, read_file, snake_case, Label, ToAbsoluteString};
		use acorn_lib::util::{download_binary, extension, make_executable, read_file, snake_case, to_absolute_string, Label, ToAbsoluteString};
		use acorn_lib::{Location, Scheme};
		use bon::Builder;
		use owo_colors::OwoColorize;
		use rayon::prelude::*;
		@@ -8,7 +9,6 @@ use std::collections::HashMap;
		use std::fs;
		use std::path::PathBuf;
		use tracing::{debug, error, info, warn};
		use uriparse::Scheme;

		use crate::script::*;

		@@ -189,15 +189,10 @@ impl Config {
		self.modules.par_iter().for_each(\|module\| match &module.module_type {
		\| ModuleType::Binary { uri, checksum } => match ModuleUri::from(uri) {
		\| ModuleUri::Url(value) if !options.offline.unwrap_or_default() => match value.scheme() {
		\| Scheme::HTTP \| Scheme::HTTPS \| Scheme::SSH => {
		\| uriparse::Scheme::HTTP \| uriparse::Scheme::HTTPS \| uriparse::Scheme::SSH => {
		let url = value.to_string();
		if ModuleUri::is_allowed(value) {
		// TODO: Implement parametrized whitelist
		debug!(module = module.name, url, "=> {} Remote binary", Label::found());
		info!(module = module.name, url, "=> {} Remote binary URL is on whitelist", Label::pass());
		} else {
		error!(module = module.name, url, "=> {} Not on whitelist", Label::rejected());
		std::process::exit(exitcode::NOPERM);
		}
		// TODO: Check that URL exists
		if !options.dry_run.unwrap_or_default() {
		let output = match options.root.clone() {
		@@ -283,15 +278,10 @@ impl Config {
		\| ModuleType::Script { programming_language, uri } => match programming_language {
		\| Some(ModuleLanguage::Generic) \| None => match ModuleUri::from(uri) {
		\| ModuleUri::Url(value) if !options.offline.unwrap_or_default() => match value.scheme() {
		\| Scheme::HTTP \| Scheme::HTTPS \| Scheme::SSH => {
		\| uriparse::Scheme::HTTP \| uriparse::Scheme::HTTPS \| uriparse::Scheme::SSH => {
		let url = value.to_string();
		if ModuleUri::is_allowed(value) {
		debug!(module = module.name, url, "=> {} Remote script repository", Label::found());
		info!(module = module.name, url, "=> {} URL is on whitelist", Label::pass());
		} else {
		error!(module = module.name, url, "=> {} Not on whitelist", Label::rejected());
		std::process::exit(exitcode::NOPERM);
		}
		// TODO: Implement parametrized whitelist
		// TODO: Check that URL exists
		// TODO: Git clone module
		// TODO: Check script repository with is_valid_xylem_module_python
		@@ -315,17 +305,13 @@ impl Config {
		},
		\| Some(ModuleLanguage::Python) => match ModuleUri::from(uri) {
		\| ModuleUri::Url(value) if !options.offline.unwrap_or_default() => match value.scheme() {
		\| Scheme::HTTP \| Scheme::HTTPS \| Scheme::SSH => {
		\| uriparse::Scheme::HTTP \| uriparse::Scheme::HTTPS \| uriparse::Scheme::SSH => {
		let url = value.to_string();
		if ModuleUri::is_allowed(value) {
		debug!(module = module.name, url, "=> {} Remote Python script repository", Label::found());
		info!(module = module.name, url, "=> {} URL is on whitelist", Label::pass());
		} else {
		error!(module = module.name, url, "=> {} Not on whitelist", Label::rejected());
		std::process::exit(exitcode::NOPERM);
		}
		// TODO: Implement parametrized whitelist
		// TODO: Check that URL exists
		if ModuleUri::from(uri).test() {
		// if Location::Simple(uri.to_string()).exists() {
		if true {
		info!(module = module.name, uri, "=> {}", Label::found());
		} else {
		error!(module = module.name, uri, "=> {}", Label::rejected());
		@@ -339,7 +325,7 @@ impl Config {
		}
		};
		let module_root = match options.root.clone() {
		\| Some(value) => value.join(ModuleUri::from(uri).hash()).to_absolute_string(),
		\| Some(value) => value.join(Location::Simple(uri.into()).hash()).to_absolute_string(),
		\| None => {
		error!(module = module.name, "=> {} Root directory", Label::not_found());
		std::process::exit(exitcode::UNAVAILABLE);
		@@ -506,7 +492,15 @@ impl Config {
		std::process::exit(exitcode::UNAVAILABLE);
		}
		};
		let working_dir = ModuleUri::from(uri).working_directory(root.clone());
		let location = Location::Simple(uri.into());
		let working_dir = match location.scheme() {
		\| Scheme::HTTPS => match root.clone() {
		\| Some(value) => value.join(location.hash()).to_absolute_string(),
		\| None => location.hash(),
		},
		\| Scheme::File => to_absolute_string(location.uri().unwrap().path().to_string()),
		\| _ => unimplemented!(),
		};
		let loaded = Script::from_module(module.clone(), root);
		let script = loaded.expand_arguments_from(envs).with_current_dir(working_dir);
		if options.dry_run.unwrap_or_default() {
		@@ -530,7 +524,15 @@ impl Config {
		std::process::exit(exitcode::UNAVAILABLE);
		}
		};
		let working_dir = ModuleUri::from(uri).working_directory(root.clone());
		let location = Location::Simple(uri.into());
		let working_dir = match location.scheme() {
		\| Scheme::HTTPS => match root.clone() {
		\| Some(value) => value.join(location.hash()).to_absolute_string(),
		\| None => location.hash(),
		},
		\| Scheme::File => to_absolute_string(location.uri().unwrap().path().to_string()),
		\| _ => unimplemented!(),
		};
		let virtual_environment = match &module.template {
		\| Some(TemplateAttribute::Path(value)) => {
		ScriptTemplate::from_path(PathBuf::from(&working_dir).join(value)).virtual_environment

xylem-lib/src/script/mod.rs

+22 −69

Original line number	Diff line number	Diff line
		use acorn_lib::util::{read_file, Label};
		use acorn_lib::util::{read_file, to_absolute_string, Label, ToAbsoluteString};
		use acorn_lib::{Location, Scheme};
		use bon::Builder;
		use derive_more::{Display, FromStr};
		use fancy_regex::Regex;
		@@ -13,9 +14,7 @@ use std::ffi::OsStr;
		use std::io::{BufRead, BufReader};
		use std::path::PathBuf;
		use std::process;
		use tokio::time::{sleep, Duration, Sleep};
		use tracing::{debug, error, info, trace, warn};
		use uriparse::Scheme;
		use uriparse::URI;
		use valuable_derive::Valuable;
		use which::which;
		@@ -396,71 +395,13 @@ impl ModuleUri<'_> {
		pub fn from(value: &str) -> ModuleUri<'_> {
		match URI::try_from(value) {
		\| Ok(uri) => match uri.scheme() {
		\| Scheme::HTTP \| Scheme::HTTPS \| Scheme::SSH => ModuleUri::Url(uri),
		\| Scheme::File => ModuleUri::File(PathBuf::from(uri.path().to_string())),
		\| uriparse::Scheme::HTTP \| uriparse::Scheme::HTTPS \| uriparse::Scheme::SSH => ModuleUri::Url(uri),
		\| uriparse::Scheme::File => ModuleUri::File(PathBuf::from(uri.path().to_string())),
		\| _ => ModuleUri::Unknown(value.to_string()),
		},
		\| Err(_) => ModuleUri::Unknown(value.to_string()),
		}
		}
		pub fn hash(&self) -> String {
		match self {
		\| ModuleUri::Url(uri) => {
		let host = match uri.host() {
		\| Some(value) => value.clone().to_string().replace('.', "_"),
		\| None => "".to_string(),
		};
		let segments = uri
		.path()
		.segments()
		.iter()
		.map(\|s\| s.to_string())
		.filter(\|s\| !s.is_empty())
		.collect::<Vec<_>>();
		if segments.is_empty() {
		host.to_string()
		} else {
		format!("{}_{}", host, segments[segments.len() - 2..].join("_").to_lowercase())
		}
		}
		\| ModuleUri::File(value) => value.clone().into_os_string().into_string().unwrap(),
		\| ModuleUri::Unknown(_) => unimplemented!(),
		}
		}
		pub fn get_whitelist() -> Vec<String> {
		vec!["code.ornl.gov", "code-int.ornl.gov"].into_iter().map(String::from).collect()
		}
		pub fn is_allowed(uri: URI) -> bool {
		ModuleUri::get_whitelist().contains(&uri.host().unwrap().to_string())
		}
		pub fn test(&self) -> bool {
		async fn check(_url: String) -> Sleep {
		sleep(Duration::from_secs(2))
		}
		match self {
		\| ModuleUri::Url(uri) => match uri.scheme() {
		\| Scheme::SSH if ModuleUri::is_allowed(uri.clone()) => true,
		\| _ if ModuleUri::is_allowed(uri.clone()) => {
		// let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
		// let _ = runtime.block_on(check(uri.to_string()));
		true
		}
		\| _ => false,
		},
		\| ModuleUri::File(value) => value.clone().exists(),
		\| ModuleUri::Unknown(_) => unimplemented!(),
		}
		}
		pub fn working_directory(&self, root: Option<PathBuf>) -> String {
		match self {
		\| ModuleUri::Url(_) => match root {
		\| Some(value) => value.join(self.hash()).into_os_string().into_string().unwrap(),
		\| None => self.hash(),
		},
		\| ModuleUri::File(value) => value.clone().into_os_string().into_string().unwrap(),
		\| ModuleUri::Unknown(value) => value.to_string(),
		}
		}
		}
		impl Script {
		pub fn ensure_relative(mut self) -> Script {
		@@ -556,11 +497,18 @@ impl Script {
		pub fn from_module(module: Module, root: Option<PathBuf>) -> Script {
		const DEFAULT_FILENAME: &str = "template.json";
		let module_uri = match module.module_type {
		\| ModuleType::Binary { ref uri, .. } => ModuleUri::from(uri),
		\| ModuleType::Script { ref uri, .. } => ModuleUri::from(uri),
		\| ModuleType::Binary { ref uri, .. } => Location::Simple(uri.into()),
		\| ModuleType::Script { ref uri, .. } => Location::Simple(uri.into()),
		\| ModuleType::Configuration { .. } => unimplemented!("Configuration module type"),
		};
		let working_dir = module_uri.working_directory(Some(root.unwrap_or_default().clone()));
		let working_dir = match root {
		\| Some(path) => match module_uri.scheme() {
		\| Scheme::HTTPS => path.join(module_uri.hash()).to_absolute_string(),
		\| Scheme::File => module_uri.uri().unwrap().to_string(),
		\| _ => module_uri.hash(),
		},
		\| None => module_uri.hash(),
		};
		let default_template_path = PathBuf::from(&working_dir).join(DEFAULT_FILENAME);
		match &module.template {
		\| Some(attribute) => match attribute {
		@@ -708,7 +656,12 @@ impl Script {
		/// Create virtual environment from associated manifest file (ex. environment.yml or pyproject.toml)
		// TODO: Add support for Pixi and other virtual environment managers
		pub fn maybe_create_virtual_environment(parent: String, _manager: Option<VirtualEnvironmentManager>) -> Option<Script> {
		let current_dir = ModuleUri::from(&parent).working_directory(None);
		let location = Location::Simple(parent);
		let current_dir = match location.scheme() {
		\| Scheme::HTTPS => location.hash(),
		\| Scheme::File => to_absolute_string(location.uri().unwrap().path().to_string()),
		\| _ => unimplemented!(),
		};
		let manifest_path = PathBuf::from(&current_dir).join("environment.yml");
		match manifest_path.try_exists() {
		\| Ok(true) => {
		@@ -749,9 +702,9 @@ impl Script {
		}
		}
		pub fn maybe_git_clone(url: String, current_dir: String) -> Option<Script> {
		let module_uri = ModuleUri::from(&url);
		let location = Location::Simple(url.clone());
		let command = Command::init().name("git".to_string()).build();
		let arguments = vec!["clone".to_string(), url.clone(), module_uri.hash()];
		let arguments = vec!["clone".to_string(), url, location.hash()];
		let script = Script::init().command(command).arguments(arguments).build().with_current_dir(current_dir);
		Some(script)
		}

xylem-lib/src/tests.rs

+0 −35

Original line number	Diff line number	Diff line
		@@ -148,41 +148,6 @@ fn test_module() {
		assert_eq!(module.to_string(), "module-a");
		}
		#[test]
		fn test_module_uri() {
		assert_eq!(ModuleUri::from(""), ModuleUri::Unknown("".to_string()));
		assert_eq!(ModuleUri::from("Not a valid URI"), ModuleUri::Unknown("Not a valid URI".to_string()));
		assert_eq!(
		ModuleUri::from("/path/to/local/folder"),
		ModuleUri::Unknown("/path/to/local/folder".to_string())
		);
		assert_eq!(ModuleUri::from("file:///etc/to/binary"), ModuleUri::File(PathBuf::from("/etc/to/binary")));
		match ModuleUri::from("https://code.ornl.gov/GSHS/GDS/Common/PIPE/module-a") {
		\| ModuleUri::Url(uri) => {
		assert_eq!(uri.scheme(), "HTTPS");
		assert_eq!(uri.path(), "/GSHS/GDS/Common/PIPE/module-a");
		}
		\| _ => panic!(),
		}
		match ModuleUri::from("ssh://git@code.ornl.gov/GSHS/GDS/Common/PIPE/module-a.git") {
		\| ModuleUri::Url(uri) => {
		assert_eq!(uri.scheme(), "SSH");
		assert_eq!(uri.path(), "/GSHS/GDS/Common/PIPE/module-a.git");
		}
		\| _ => panic!(),
		}
		}
		#[test]
		fn test_module_uri_hash() {
		let expected = "code_ornl_gov_xylem_module-a";
		let url = ModuleUri::from("http://code.ornl.gov/xylem/module-a");
		assert_eq!(expected, url.hash());
		let url = ModuleUri::from("https://code.ornl.gov/research-enablement/xylem/module-a");
		assert_eq!(expected, url.hash());
		let expected = "example_com";
		let url = ModuleUri::from("http://example.com");
		assert_eq!(expected, url.hash());
		}
		#[test]
		fn test_read_pyproject_toml() {
		let path = format!("{FIXTURES}/pyproject.toml");
		match PyProjectToml::read(PathBuf::from(path)) {