feat: Remove duct dep and refactor out tokio blocking contexts (37abfe4f) · Commits · Research Enablement / ACORN

Cargo.lock

+0 −40

Original line number	Diff line number	Diff line
		@@ -16,7 +16,6 @@ dependencies = [
		"data-encoding",
		"derive_more",
		"dotenvy",
		"duct",
		"exitcode",
		"fancy-regex",
		"fast_qr",
		@@ -70,7 +69,6 @@ dependencies = [
		"derive_more",
		"directories",
		"dotenvy",
		"duct",
		"exitcode",
		"fancy-regex",
		"flate2",
		@@ -1547,18 +1545,6 @@ version = "0.11.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"

		[[package]]
		name = "duct"
		version = "1.1.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d7478638a31d1f1f3d6c9f5e57c76b906a04ac4879d6fd0fb6245bc88f73fd0b"
		dependencies = [
		"libc",
		"os_pipe",
		"shared_child",
		"shared_thread",
		]

		[[package]]
		name = "dunce"
		version = "1.0.5"
		@@ -3492,16 +3478,6 @@ dependencies = [
		"hashbrown 0.14.5",
		]

		[[package]]
		name = "os_pipe"
		version = "1.2.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982"
		dependencies = [
		"libc",
		"windows-sys 0.59.0",
		]

		[[package]]
		name = "overload"
		version = "0.1.1"
		@@ -5415,22 +5391,6 @@ dependencies = [
		"lazy_static",
		]

		[[package]]
		name = "shared_child"
		version = "1.1.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "1e362d9935bc50f019969e2f9ecd66786612daae13e8f277be7bfb66e8bed3f7"
		dependencies = [
		"libc",
		"windows-sys 0.60.2",
		]

		[[package]]
		name = "shared_thread"
		version = "0.2.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "52b86057fcb5423f5018e331ac04623e32d6b5ce85e33300f92c79a1973928b0"

		[[package]]
		name = "shellexpand"
		version = "3.1.1"

acorn-cli/Cargo.toml

+0 −1

Original line number	Diff line number	Diff line
		@@ -27,7 +27,6 @@ clap-verbosity-flag = "3.0.2"
		color-eyre = { workspace = true }
		data-encoding = { workspace = true }
		derive_more = { workspace = true }
		duct = { version = "1.1.0", default-features = false }
		dotenvy = { workspace = true }
		exitcode = { version = "1.1.2", default-features = false }
		fancy-regex = { workspace = true }

acorn-lib/Cargo.toml

+1 −2

Original line number	Diff line number	Diff line
		@@ -38,7 +38,6 @@ data-encoding = { workspace = true }
		derive_more = { workspace = true }
		directories = { version = "6.0.0", default-features = false }
		dotenvy = { workspace = true }
		duct = { version = "1.1.0", default-features = false, optional = true }
		fancy-regex = { workspace = true }
		flate2 = { version = "1.1.5", default-features = false }
		futures = { workspace = true }
		@@ -95,7 +94,7 @@ analyzer = ["dep:bat"]
		default = ["analyzer", "std"]
		doctor = []
		powerpoint = []
		std = ["dep:duct", "dep:lychee-lib", "dep:reqwest", "dep:tokio", "dep:which"]
		std = ["dep:lychee-lib", "dep:reqwest", "dep:tokio", "dep:which"]

		[lints.rust]
		missing-docs = "deny"

acorn-lib/src/analyzer/mod.rs

+59 −49

Original line number	Diff line number	Diff line
		@@ -7,7 +7,7 @@ use crate::io::{command_exists, download_binary, extract_zip, file_checksum, mak
		#[cfg(feature = "std")]
		use crate::io::{get, InputOutput};
		use crate::prelude;
		use crate::prelude::{create_dir_all, remove_file, Error, File, HashMap, PathBuf, Write};
		use crate::prelude::{create_dir_all, remove_file, Command, Error, File, HashMap, PathBuf, Write};
		use crate::schema::research_activity::ResearchActivity;
		use crate::schema::{ImageObject, MediaObject, Organization, ProgrammingLanguage, VideoObject, Website};
		use crate::skip;
		@@ -25,7 +25,6 @@ use color_eyre::owo_colors::OwoColorize;
		use convert_case::{Case, Casing};
		use core::ops::RangeInclusive;
		use derive_more::Display;
		use duct::cmd;
		use flate2::read::GzDecoder;
		use futures::future::join_all;
		use ini::Ini;
		@@ -35,7 +34,6 @@ use polars::frame::row::Row;
		use polars::prelude::{AnyValue, DataFrame, PolarsResult};
		use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
		use tar::Archive;
		use tokio;
		use tracing::{debug, error, info, trace};
		use validator::Validate;
		use validator::ValidationErrorsKind;
		@@ -80,9 +78,9 @@ pub trait StaticAnalyzer<Config: StaticAnalyzerConfig> {
		/// Resolve analyzer
		async fn resolve(_config: Config, _is_offline: bool, _skip_verify_checksum: bool) -> Self;
		/// Run analyzer on content
		fn run(&self, id: String, content: String, output: Option<String>) -> Check;
		async fn run(&self, id: String, content: String, output: Option<String>) -> Check;
		/// Perform sync operation (only applies to Vale)
		fn sync(self, is_offline: bool) -> Result<(), Error>;
		async fn sync(self, is_offline: bool) -> Result<(), Error>;
		/// Set binary
		fn with_binary<P>(self, path: P) -> Self
		where
		@@ -199,22 +197,21 @@ impl Analysis for ResearchActivity {
		// Create and save Vale config asynchronously
		let config = ValeConfig::default().save().await;
		let vale = Vale::resolve(config, is_offline, skip_verify_checksum).await;
		let vale_clone = vale.clone();
		match tokio::task::spawn_blocking(move \|\| vale_clone.sync(is_offline)).await.unwrap() {
		match vale.clone().sync(is_offline).await {
		\| Ok(_) => {
		let mut results = Vec::new();
		for path in paths {
		let vale_clone = vale.clone();
		let path_clone = path.clone();
		let result = tokio::task::spawn_blocking(move \|\| match ResearchActivity::read(&path_clone) {
		\| Ok(data) => vale_clone.run(data.clone().meta.identifier, data.to_markdown(), Some("JSON".into())),
		let result = match ResearchActivity::read(&path) {
		\| Ok(data) => {
		vale.clone()
		.run(data.clone().meta.identifier, data.to_markdown(), Some("JSON".into()))
		.await
		}
		\| Err(why) => {
		error!("=> {} Read research activity data - {why}", Label::fail());
		Check::init().category(CheckCategory::Prose).success(false).build()
		}
		})
		.await
		.unwrap();
		};

		results.push(result);
		}
		@@ -479,7 +476,7 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		let init = Vale::init().build();
		let vale = if command_exists(name) {
		let init_with_config = init.with_config(config);
		tokio::task::spawn_blocking(move \|\| init_with_config.with_system_command()).await.unwrap()
		init_with_config.with_system_command()
		} else if is_offline \|\| any_exist(vec![format!("{root}{name}"), format!("{root}{name}.exe")]) {
		info!("=> {} Local {} binary", Label::using(), name.green().bold());
		#[cfg(any(unix, target_os = "wasi", target_os = "redox"))]
		@@ -495,7 +492,7 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		};
		vale
		}
		fn run(&self, id: String, content: String, output: Option<String>) -> Check {
		async fn run(&self, id: String, content: String, output: Option<String>) -> Check {
		let root = standard_project_folder("check", None);
		match create_dir_all(root.clone()) {
		\| Ok(_) => {}
		@@ -518,34 +515,31 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		match &self.config {
		\| Some(config) => {
		let result = match output {
		\| Some(value) => cmd!(
		binary,
		"--no-wrap",
		"--config",
		config.clone().path,
		"--output",
		value,
		path.clone(),
		"--ext",
		".md",
		"--no-exit",
		)
		.read(),
		\| None => cmd!(
		binary,
		"--no-wrap",
		"--config",
		config.clone().path,
		path.clone(),
		"--ext",
		".md",
		"--no-exit"
		)
		.read(),
		\| Some(value) => Command::new(binary)
		.arg("--no-wrap")
		.arg("--config")
		.arg(config.clone().path)
		.arg("--output")
		.arg(value)
		.arg(path.clone())
		.arg("--ext")
		.arg(".md")
		.arg("--no-exit")
		.output(),
		\| None => Command::new(binary)
		.arg("--no-wrap")
		.arg("--config")
		.arg(config.clone().path)
		.arg(path.clone())
		.arg("--ext")
		.arg(".md")
		.arg("--no-exit")
		.output(),
		};
		match result {
		\| Ok(output) => {
		let parsed = ValeOutput::parse(&output, path.clone());
		\| Ok(output) if output.status.success() => {
		let stdout = String::from_utf8_lossy(&output.stdout).to_string();
		let parsed = ValeOutput::parse(&stdout, path.clone());
		if parsed.is_empty() {
		Check::init().category(CheckCategory::Prose).success(true).message(id).build()
		} else {
		@@ -559,8 +553,18 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		.build()
		}
		}
		\| Err(output) => {
		error!("=> {} Analyze - {}", Label::fail(), output);
		\| Ok(output) => {
		let why = String::from_utf8_lossy(&output.stderr).trim().to_string();
		let message = if why.is_empty() {
		format!("process exited with status {}", output.status)
		} else {
		why
		};
		error!("=> {} Analyze - {}", Label::fail(), message);
		Check::init().category(CheckCategory::Prose).success(false).message(id).build()
		}
		\| Err(why) => {
		error!("=> {} Analyze - {}", Label::fail(), why);
		Check::init().category(CheckCategory::Prose).success(false).message(id).build()
		}
		}
		@@ -693,7 +697,7 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		}
		}
		}
		fn sync(self, is_offline: bool) -> Result<(), Error> {
		async fn sync(self, is_offline: bool) -> Result<(), Error> {
		let command = self.clone().command();
		let path = match self.binary {
		\| Some(value) => value,
		@@ -703,11 +707,15 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		}
		};
		let config_path = self.config.unwrap().path;
		let result = if is_offline {
		let result: Result<(), Error> = if is_offline {
		skip!("Vale sync");
		cmd!("").run()
		Ok(())
		} else {
		cmd!(path.clone(), "--config", config_path.clone(), "sync").run()
		match Command::new(path.clone()).arg("--config").arg(config_path.clone()).arg("sync").status() {
		\| Ok(status) if status.success() => Ok(()),
		\| Ok(status) => Err(Error::other(format!("Vale sync failed with status: {status}"))),
		\| Err(why) => Err(why),
		}
		};
		match result {
		\| Ok(_) => {
		@@ -767,7 +775,9 @@ impl StaticAnalyzer<ValeConfig> for Vale {
		let path = which(name.clone()).unwrap().to_path_buf();
		self.binary = Some(path.clone());
		let offset = "vale version ".len();
		let version = cmd!(name.clone(), "--version").read().unwrap()[offset..].to_string();
		let output = Command::new(name.clone()).arg("--version").output().unwrap();
		let stdout = String::from_utf8(output.stdout).unwrap();
		let version = stdout[offset..].trim().to_string();
		self.version = Some(SemanticVersion::from(version.as_ref()));
		debug!(
		path = path.to_absolute_string(),

acorn-lib/src/io/mod.rs

+44 −21

Original line number	Diff line number	Diff line
		@@ -20,7 +20,7 @@
		//! ```
		//!
		use crate::fail;
		use crate::prelude::{canonicalize, create_dir_all, io, var, BufReader, Cursor, Error, File, PathBuf, Read, Write};
		use crate::prelude::{canonicalize, create_dir_all, io, var, BufReader, Command, Cursor, Error, File, PathBuf, Read, Write};
		#[cfg(any(unix, target_os = "wasi", target_os = "redox"))]
		use crate::prelude::{set_permissions, Permissions, PermissionsExt};
		use crate::util::constants::{APPLICATION, LARGE_FILE_THRESHOLD_BYTES, ORGANIZATION, QUALIFIER};
		@@ -30,7 +30,6 @@ use crate::util::{generate_guid, suffix, Label, MimeType, SemanticVersion, ToAbs
		use core::time::Duration;
		use data_encoding::HEXUPPER;
		use directories::ProjectDirs;
		use duct::cmd;
		use fancy_regex::Regex;
		use glob::glob;
		use is_executable::IsExecutable;
		@@ -60,7 +59,7 @@ pub trait FromCommand {
		fn from_command<S>(name: S) -> Option<Self>
		where
		Self: Sized,
		S: Into<String> + duct::IntoExecutablePath + core::marker::Copy;
		S: Into<String> + core::marker::Copy;
		}
		/// Add `from_path` trait to a value (like `MimeType`)
		pub trait FromPath {
		@@ -158,19 +157,19 @@ impl FromCommand for SemanticVersion {
		#[cfg(feature = "std")]
		fn from_command<S>(name: S) -> Option<SemanticVersion>
		where
		S: Into<String> + duct::IntoExecutablePath + core::marker::Copy,
		S: Into<String> + core::marker::Copy,
		{
		if command_exists(name.into()) {
		let result = cmd(name, vec!["--version"]).read();
		match result {
		\| Ok(value) => {
		let first_line = value.lines().collect::<Vec<_>>().first().cloned();
		match first_line {
		let command = name.into();
		if command_exists(command.clone()) {
		match Command::new(&command).arg("--version").output() {
		\| Ok(output) if output.status.success() => {
		let value = String::from_utf8_lossy(&output.stdout);
		match value.lines().next() {
		\| Some(line) => Some(SemanticVersion::from(line)),
		\| None => None,
		}
		}
		\| Err(_) => None,
		\| Ok(_) \| Err(_) => None,
		}
		} else {
		None
		@@ -524,9 +523,19 @@ pub fn files_from_git_branch(value: &str, extensions: Option<Vec<&str>>) -> Vec<
		\| None => "main".to_string(),
		};
		let args = vec!["diff", "--name-only", &default_branch, "--merge-base", value];
		let result = cmd("git", args).read();
		let result = Command::new("git").args(args).output();
		match result {
		\| Ok(value) => filter_git_command_result(value, extensions),
		\| Ok(output) if output.status.success() => filter_git_command_result(String::from_utf8_lossy(&output.stdout).to_string(), extensions),
		\| Ok(output) => {
		let why = String::from_utf8_lossy(&output.stderr).trim().to_string();
		let message = if why.is_empty() {
		format!("process exited with status {}", output.status)
		} else {
		why
		};
		error!("=> {} Get files from Git branch - {}", Label::fail(), message);
		vec![]
		}
		\| Err(why) => {
		error!("=> {} Get files from Git branch - {why}", Label::fail());
		vec![]
		@@ -545,10 +554,20 @@ pub fn files_from_git_branch(value: &str, extensions: Option<Vec<&str>>) -> Vec<
		pub fn files_from_git_commit(value: &str, extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
		if command_exists("git".to_owned()) {
		let args = vec!["diff-tree", "--no-commit-id", "--name-only", "-r", value];
		let result = cmd("git", args).read();
		let result = Command::new("git").args(args).output();
		debug!("=> {} Git command response - {result:?}", Label::using());
		let files = match result {
		\| Ok(value) => filter_git_command_result(value, extensions),
		\| Ok(output) if output.status.success() => filter_git_command_result(String::from_utf8_lossy(&output.stdout).to_string(), extensions),
		\| Ok(output) => {
		let why = String::from_utf8_lossy(&output.stderr).trim().to_string();
		let message = if why.is_empty() {
		format!("process exited with status {}", output.status)
		} else {
		why
		};
		error!("=> {} Get files from Git commit - {}", Label::fail(), message);
		vec![]
		}
		\| Err(why) => {
		error!("=> {} Get files from Git commit - {why}", Label::fail());
		vec![]
		@@ -663,15 +682,17 @@ pub fn folder_size<P: Into<PathBuf>>(path: P) -> u64 {
		pub fn git_branch_name() -> Option<String> {
		if command_exists("git".to_owned()) {
		let args = vec!["symbolic-ref", "--short", "HEAD"];
		let result = cmd("git", args).read();
		let result = Command::new("git").args(args).output();
		match result {
		\| Ok(ref value) => {
		let name = match value.clone().split("/").last() {
		\| Ok(output) if output.status.success() => {
		let value = String::from_utf8_lossy(&output.stdout).to_string();
		let name = match value.split("/").last() {
		\| Some(x) => Some(x.to_string()),
		\| None => None,
		};
		name
		}
		\| Ok(_) => None,
		\| Err(_) => None,
		}
		} else {
		@@ -686,15 +707,17 @@ pub fn git_branch_name() -> Option<String> {
		pub fn git_default_branch_name() -> Option<String> {
		if command_exists("git".to_owned()) {
		let args = vec!["symbolic-ref", "refs/remotes/origin/HEAD", "--short"];
		let result = cmd("git", args).read();
		let result = Command::new("git").args(args).output();
		match result {
		\| Ok(ref value) => {
		let name = match value.clone().split("/").last() {
		\| Ok(output) if output.status.success() => {
		let value = String::from_utf8_lossy(&output.stdout).to_string();
		let name = match value.split("/").last() {
		\| Some(x) => Some(x.to_string()),
		\| None => None,
		};
		name
		}
		\| Ok(_) => None,
		\| Err(_) => None,
		}
		} else {