4 Commits

Author SHA1 Message Date
orrisroot 459dd1cd7c chore(release): sync lockfile version
Release / build-linux-x86_64 (push) Successful in 2m16s
Release / build-linux-aarch64 (push) Successful in 2m3s
Record the Rust package version bump in Cargo.lock so the
repository stays consistent after updating the crate version
to 2.0.0.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-20 18:19:06 +09:00
orrisroot 9d29aad463 chore(release): bump version to 2.0.0
Update the Rust package manifest to 2.0.0 so CLI version
reporting and release-related flows pick up the new version
from Cargo metadata.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-20 18:18:06 +09:00
orrisroot d25ab69d13 perf(transfer): parallelize folder traversal API calls
Use a shared API request limiter across recursive upload and
download traversal so folder detail fetches, file listings,
folder auth, and transfers can run concurrently under one
budget.

Refactor the traversal loops into task-driven pipelines while
preserving skip-if-exists, excludes, cleanup, and current
output behavior.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-20 17:25:14 +09:00
orrisroot 14991b18fb perf(cache): reuse auth state in memory
Cache parsed auth state per remote and validate it with on-disk\nfile metadata so repeated authenticated API calls can skip\nredundant open/read/JSON parse work within one process.\n\nCentralize cache load, persist, and removal helpers in the cache\nmodule, reuse them from login, logout, and whoami, and update\nthe refresh path to persist structured cache data directly.\n\nAdd targeted cache tests for memory reuse, invalidation after\nexternal writes, persist updates, and cache removal.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-20 16:51:26 +09:00
15 changed files with 462 additions and 1101 deletions
Generated
+396 -235
View File
File diff suppressed because it is too large Load Diff
+10 -10
View File
@@ -1,6 +1,6 @@
[package]
name = "mdrs-client-rust"
version = "2.0.1"
version = "2.0.0"
edition = "2024"
license = "MIT"
authors = ["Neuroinformatics Unit, RIKEN CBS"]
@@ -14,23 +14,23 @@ path = "src/main.rs"
clap = { version = "4.5", features = ["derive"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.150"
tokio = { version = "1.52.3", features = ["full"] }
serde_json = "1.0"
tokio = { version = "1.37", features = ["full"] }
futures = "0.3"
dirs = "6.0.0"
dirs = "5.0"
anyhow = "1.0.102"
configparser = "3.2.0"
configparser = "3.1.0"
validators = "0.25.3"
sha2 = "0.11.0"
rpassword = "7.5.4"
sha2 = "0.10"
rpassword = "7.0"
base64 = "0.22"
fs2 = "0.4"
ctrlc = "3"
os_info = "3.15.0"
os_info = "3"
dotenvy = "0.15"
unicode-normalization = "0.1"
self-replace = "1"
tar = "0.4.46"
tar = "0.4"
flate2 = "1"
zip = "8.6.0"
zip = "2"
tempfile = "3"
+5 -28
View File
@@ -103,7 +103,7 @@ mdrs labs neurodata:
### ls
List the contents of a remote folder. You can also specify a DOI path in the form `remote:10.xxxx/yyy.ID[/optional/subpath]`.
List the contents of a remote folder.
```shell
mdrs ls neurodata:/NIU/Repository/
@@ -111,10 +111,6 @@ mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
mdrs ls -r neurodata:/NIU/Repository/Dataset1/
mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
mdrs ls -q neurodata:/NIU/Repository/
# DOI access examples:
mdrs ls neurodata:10.60178/cbs.20260429-001
mdrs ls "neurodata:10.60178/cbs.20260429-001/Figure 1"
```
### mkdir
@@ -137,7 +133,7 @@ mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
### download
Download a file or folder from a remote path. You can also specify a DOI path.
Download a file or folder from a remote path.
```shell
mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
@@ -145,9 +141,6 @@ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
# DOI access examples:
mdrs download -r neurodata:10.60178/cbs.20260429-001 ./
```
### mv
@@ -191,26 +184,20 @@ Available access levels: `private`, `public`, `pw_open`, `cbs_open`, `5kikan_ope
### metadata
Show metadata for a remote folder. You can also specify a DOI path.
Show metadata for a remote folder.
```shell
mdrs metadata neurodata:/NIU/Repository/Private/
mdrs metadata neurodata:/NIU/Repository/TEST/
mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
# DOI access examples:
mdrs metadata neurodata:10.60178/cbs.20260429-001
```
### file-metadata
Show metadata for a remote file. You can also specify a DOI path.
Show metadata for a remote file.
```shell
mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
# DOI access examples:
mdrs file-metadata "neurodata:10.60178/cbs.20260429-001/Figure 1/Figure1v3.pdf"
```
### version
@@ -221,16 +208,6 @@ Show the tool name and version number.
mdrs version
```
### selfupdate
Update the current `mdrs` binary to the latest published release for
the same build target.
```shell
mdrs selfupdate
mdrs selfupdate -y
```
### help
Show help for a command.
-17
View File
@@ -1,17 +0,0 @@
use crate::connection::MDRSConnection;
use crate::models::doi::DoiResponse;
use anyhow::bail;
impl MDRSConnection {
/// Retrieve the folder associated with a DOI suffix ID (GET v3/doi/{id}/).
///
/// The MDRS DOI format is `10.xxxx/prefix.{id}` where the suffix after the
/// last `.` is the internal system ID passed to this endpoint.
pub async fn retrieve_doi(&self, id: &str) -> Result<DoiResponse, anyhow::Error> {
let resp = self.get(&format!("v3/doi/{}/", id)).await?;
if !resp.status().is_success() {
bail!("DOI lookup failed: {}", resp.status());
}
Ok(resp.json::<DoiResponse>().await?)
}
}
-1
View File
@@ -1,6 +1,5 @@
// API module (add users, files, folders, laboratories, etc. here)
pub mod doi;
pub mod files;
pub mod folders;
pub mod laboratories;
+1 -2
View File
@@ -119,6 +119,5 @@ pub fn compute_digest(
let json_str = python_digest_json(user, access, refresh, labs);
let mut hasher = Sha256::new();
hasher.update(json_str.as_bytes());
let result = hasher.finalize();
result.iter().map(|b| format!("{:02x}", b)).collect()
format!("{:x}", hasher.finalize())
}
+4 -169
View File
@@ -185,43 +185,6 @@ fn load_cache_from_dir(remote: &str, config_dir: &Path) -> Result<Cache, anyhow:
Ok(cache)
}
fn load_cache_if_present_from_dir(
remote: &str,
config_dir: &Path,
) -> Result<Option<Cache>, anyhow::Error> {
let cache_path = cache_file_path_in(config_dir, remote);
let snapshot = match read_cache_snapshot(&cache_path) {
Ok(snapshot) => snapshot,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
return Ok(None);
}
Err(e) => return Err(e.into()),
};
if let Some(cache) = cached_entry(config_dir, remote, &snapshot) {
return Ok(Some(cache));
}
let data = match fs::read_to_string(&cache_path) {
Ok(data) => data,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
return Ok(None);
}
Err(e) => return Err(e.into()),
};
let cache = match parse_cache(remote, &data) {
Ok(cache) => cache,
Err(_) => {
remove_cache_in_dir(remote, config_dir)?;
return Ok(None);
}
};
update_cached_entry(config_dir, remote, snapshot, cache.clone());
Ok(Some(cache))
}
fn persist_cache_in_dir(
remote: &str,
config_dir: &Path,
@@ -284,7 +247,6 @@ pub async fn load_cache_with_token_refresh(remote: &str) -> Result<Cache, anyhow
let lock = get_remote_lock(remote);
let _guard = lock.lock().await;
ensure_cache_dir(&cache_dir_path(&crate::settings::SETTINGS.config_dirname))?;
let lock_path = cache_file_path(remote).with_extension("lock");
use fs2::FileExt;
let lock_file = fs::OpenOptions::new()
@@ -318,71 +280,9 @@ pub async fn load_cache_with_token_refresh(remote: &str) -> Result<Cache, anyhow
result
}
async fn load_cache_with_token_refresh_optional_from_dir(
remote: &str,
config_dir: &Path,
) -> Result<Option<Cache>, anyhow::Error> {
let lock = get_remote_lock(remote);
let _guard = lock.lock().await;
ensure_cache_dir(&cache_dir_path(config_dir))?;
let lock_path = cache_file_path_in(config_dir, remote).with_extension("lock");
use fs2::FileExt;
let lock_file = fs::OpenOptions::new()
.write(true)
.create(true)
.open(&lock_path)?;
lock_file.lock_exclusive()?;
let result: Result<Option<Cache>, anyhow::Error> = async {
let Some(mut cache) = load_cache_if_present_from_dir(remote, config_dir)? else {
return Ok(None);
};
if crate::token::is_expired(&cache.token.refresh) {
remove_cache_in_dir(remote, config_dir)?;
return Ok(None);
}
if crate::token::is_refresh_required(&cache.token.access, &cache.token.refresh) {
cache = refresh_and_persist_in_dir(remote, config_dir, &cache).await?;
}
Ok(Some(cache))
}
.await;
lock_file.unlock()?;
result
}
/// Load cache when present and refresh its token if needed.
///
/// Unlike `load_cache_with_token_refresh`, this returns `Ok(None)` when the user
/// is effectively anonymous: no cache file exists, the cache is invalid, or the
/// refresh token has already expired. This mirrors the Python client behavior
/// used by read-only commands.
pub async fn load_cache_with_token_refresh_optional(
remote: &str,
) -> Result<Option<Cache>, anyhow::Error> {
load_cache_with_token_refresh_optional_from_dir(
remote,
&crate::settings::SETTINGS.config_dirname,
)
.await
}
/// Call the token-refresh endpoint and write the new access token back to the
/// cache file. The caller must already hold the per-remote async mutex.
async fn refresh_and_persist(remote: &str, cache: &Cache) -> Result<Cache, anyhow::Error> {
refresh_and_persist_in_dir(remote, &crate::settings::SETTINGS.config_dirname, cache).await
}
async fn refresh_and_persist_in_dir(
remote: &str,
config_dir: &Path,
cache: &Cache,
) -> Result<Cache, anyhow::Error> {
let url = crate::commands::config::get_remote_url(remote)?
.ok_or_else(|| anyhow!("Remote `{}` is not configured.", remote))?;
let conn = MDRSConnection::new(&url);
@@ -398,7 +298,7 @@ async fn refresh_and_persist_in_dir(
&updated_cache.laboratories,
);
persist_cache_in_dir(remote, config_dir, &updated_cache)?;
persist_cache(remote, &updated_cache)?;
Ok(updated_cache)
}
@@ -412,26 +312,11 @@ pub fn create_authenticated_conn(
remote: &str,
cache: &Cache,
) -> Result<MDRSConnection, anyhow::Error> {
Ok(create_remote_conn(remote)?.with_token(cache.token.access.clone()))
}
/// Create an unauthenticated `MDRSConnection` for the given remote label.
pub fn create_remote_conn(remote: &str) -> Result<MDRSConnection, anyhow::Error> {
let url = crate::commands::config::get_remote_url(remote)?
.ok_or_else(|| anyhow!("Remote `{}` is not configured.", remote))?;
Ok(MDRSConnection::new(&url).with_remote(remote))
}
/// Create a connection for read-only commands, attaching a bearer token only
/// when a valid login cache is available.
pub async fn create_readonly_conn(
remote: &str,
) -> Result<(MDRSConnection, Option<Cache>), anyhow::Error> {
let conn = create_remote_conn(remote)?;
match load_cache_with_token_refresh_optional(remote).await? {
Some(cache) => Ok((conn.with_token(cache.token.access.clone()), Some(cache))),
None => Ok((conn, None)),
}
Ok(MDRSConnection::new(&url)
.with_remote(remote)
.with_token(cache.token.access.clone()))
}
#[cfg(test)]
@@ -553,54 +438,4 @@ mod tests {
.contains(&format!("Not logged in to `{remote}`"))
);
}
fn make_jwt_with_exp(exp: i64) -> String {
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#);
let payload = URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#));
format!("{header}.{payload}.")
}
#[test]
fn load_cache_if_present_returns_none_when_cache_missing() {
let dir = tempdir().unwrap();
let remote = remote_name("missing", dir.path());
let loaded = load_cache_if_present_from_dir(&remote, dir.path()).unwrap();
assert!(loaded.is_none());
}
#[test]
fn load_cache_if_present_clears_invalid_cache() {
let dir = tempdir().unwrap();
let remote = remote_name("invalid", dir.path());
let cache_dir = cache_dir_path(dir.path());
ensure_cache_dir(&cache_dir).unwrap();
let cache_path = cache_file_path_in(dir.path(), &remote);
fs::write(&cache_path, b"{invalid json").unwrap();
let loaded = load_cache_if_present_from_dir(&remote, dir.path()).unwrap();
assert!(loaded.is_none());
assert!(!cache_path.exists());
}
#[tokio::test]
async fn optional_cache_load_treats_expired_session_as_anonymous() {
let dir = tempdir().unwrap();
let remote = remote_name("expired", dir.path());
let mut cache = sample_cache("alice");
cache.token.access = make_jwt_with_exp(0);
cache.token.refresh = make_jwt_with_exp(0);
persist_cache_in_dir(&remote, dir.path(), &cache).unwrap();
let loaded = load_cache_with_token_refresh_optional_from_dir(&remote, dir.path())
.await
.unwrap();
assert!(loaded.is_none());
assert!(!cache_file_path_in(dir.path(), &remote).exists());
}
}
+7 -194
View File
@@ -1,7 +1,7 @@
use crate::cache::create_readonly_conn;
use crate::cache::{create_authenticated_conn, load_cache_with_token_refresh};
use crate::commands::shared::{
find_file_by_name, find_folder_by_doi, find_folder_limited, find_laboratory,
find_subfolder_by_name, is_doi, parse_doi_remote_path, parse_remote_path,
find_file_by_name, find_folder_limited, find_lab_in_cache, find_subfolder_by_name,
parse_remote_path,
};
use crate::connection::{ApiRequestLimiter, MDRSConnection};
use anyhow::{anyhow, bail};
@@ -17,7 +17,11 @@ pub async fn download(
password: Option<&str>,
excludes: Vec<String>,
) -> Result<(), anyhow::Error> {
let (remote, labname, r_path) = parse_remote_path(remote_path)?;
let cache = load_cache_with_token_refresh(&remote).await?;
let conn = Arc::new(create_authenticated_conn(&remote, &cache)?);
let limiter = ApiRequestLimiter::new(crate::settings::SETTINGS.concurrent);
let lab = find_lab_in_cache(&cache, &labname)?;
// Validate that local_path is an existing directory (matching Python's behaviour).
let local_real = std::fs::canonicalize(local_path)
@@ -26,197 +30,6 @@ pub async fn download(
bail!("Local directory `{}` not found.", local_path);
}
// Detect DOI path: "remote:10.xxxx/prefix.ID[/optional/sub/path]"
if is_doi(remote_path.splitn(2, ':').nth(1).unwrap_or("")) {
let (remote, doi, subpath) = parse_doi_remote_path(remote_path)?;
let (raw_conn, _cache) = create_readonly_conn(&remote).await?;
let (doi_folder, lab) = find_folder_by_doi(&raw_conn, &doi, password).await?;
let abs_path = format!("{}{}", doi_folder.path.trim_end_matches('/'), subpath);
let abs_path_clean = abs_path.trim_end_matches('/');
// Check if the target is a folder.
// If it is, we download the directory. If it is not, we download the file or subfolder.
let (folder, is_folder) = match find_folder_limited(
&raw_conn,
&limiter,
lab.id,
abs_path_clean,
password,
)
.await
{
Ok(f) => (f, true),
Err(_) => {
let (parent_path, _) = match abs_path_clean.rfind('/') {
Some(0) => ("/".to_string(), abs_path_clean[1..].to_string()),
Some(pos) => (
abs_path_clean[..pos].to_string(),
abs_path_clean[pos + 1..].to_string(),
),
None => ("/".to_string(), abs_path_clean.to_string()),
};
let parent =
find_folder_limited(&raw_conn, &limiter, lab.id, &parent_path, password)
.await?;
(parent, false)
}
};
let conn = Arc::new(raw_conn);
let excludes = Arc::new(excludes);
if is_folder {
let path = folder.path.clone();
let lab_name = Arc::new(lab.name);
let top_local = local_real.join(&folder.name);
let password_owned = password.map(str::to_string);
let mut folder_tasks: JoinSet<Result<DownloadFolderTaskResult, anyhow::Error>> =
JoinSet::new();
let mut download_tasks: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
let mut errors = Vec::new();
if !recursive {
// Single-folder, non-recursive: download the files inside the DOI folder.
let files = conn.list_all_files_limited(&folder.id, &limiter).await?;
for file in &files {
if is_excluded(&excludes, lab_name.as_str(), &path, Some(&file.name)) {
continue;
}
let dest = local_real.join(&file.name);
if skip_if_exists {
if dest.exists() {
if let Ok(meta) = std::fs::metadata(&dest) {
if meta.len() == file.size {
println!("{}", dest.display());
continue;
}
}
}
}
let url = make_absolute_url(&conn, &file.download_url);
conn.download_file_limited(&url, &dest.to_string_lossy(), &limiter)
.await?;
println!("{}", dest.display());
}
return Ok(());
}
spawn_download_folder_task(
&mut folder_tasks,
conn.clone(),
limiter.clone(),
lab_name.clone(),
excludes.clone(),
folder.id.clone(),
top_local,
password_owned.clone(),
skip_if_exists,
);
drive_download_tasks(
&mut folder_tasks,
&mut download_tasks,
&mut errors,
conn.clone(),
limiter,
lab_name,
excludes,
password_owned,
skip_if_exists,
)
.await;
if !errors.is_empty() {
bail!(errors.join("\n"));
}
return Ok(());
}
// Case: target is a file or subfolder inside the resolved folder.
let basename = match abs_path_clean.rfind('/') {
Some(pos) => abs_path_clean[pos + 1..].to_string(),
None => abs_path_clean.to_string(),
};
let files = conn.list_all_files_limited(&folder.id, &limiter).await?;
// Case 1: basename matches a file in the folder.
if let Some(file) = find_file_by_name(&files, &basename) {
if is_excluded(&excludes, &lab.name, &folder.path, Some(&file.name)) {
return Ok(());
}
let dest = local_real.join(&file.name);
if skip_if_exists {
if dest.exists() {
if let Ok(meta) = std::fs::metadata(&dest) {
if meta.len() == file.size {
println!("{}", dest.display());
return Ok(());
}
}
}
}
let url = make_absolute_url(&conn, &file.download_url);
conn.download_file_limited(&url, &dest.to_string_lossy(), &limiter)
.await?;
println!("{}", dest.display());
return Ok(());
}
// Case 2: basename matches a sub-folder.
let subfolder = find_subfolder_by_name(&folder.sub_folders, &basename);
if let Some(sub) = subfolder {
if !recursive {
bail!("Cannot download `{}`: Is a folder.", abs_path_clean);
}
let top_local = local_real.join(&sub.name);
let mut folder_tasks: JoinSet<Result<DownloadFolderTaskResult, anyhow::Error>> =
JoinSet::new();
let mut download_tasks: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
let mut errors = Vec::new();
let lab_name = Arc::new(lab.name.clone());
let password_owned = password.map(str::to_string);
spawn_download_folder_task(
&mut folder_tasks,
conn.clone(),
limiter.clone(),
lab_name.clone(),
excludes.clone(),
sub.id.clone(),
top_local,
password_owned.clone(),
skip_if_exists,
);
drive_download_tasks(
&mut folder_tasks,
&mut download_tasks,
&mut errors,
conn.clone(),
limiter,
lab_name,
excludes,
password_owned,
skip_if_exists,
)
.await;
if !errors.is_empty() {
bail!(errors.join("\n"));
}
return Ok(());
}
bail!("File or folder `{}` not found.", abs_path_clean);
}
// Normal path: "remote:/labname/path/..."
let (remote, labname, r_path) = parse_remote_path(remote_path)?;
let (raw_conn, cache) = create_readonly_conn(&remote).await?;
let conn = Arc::new(raw_conn);
let lab = find_laboratory(&conn, cache.as_ref(), &labname).await?;
// Split r_path into the parent directory path and the target basename.
// Trailing slashes are already stripped by parse_remote_path, so this is safe.
let r_path_clean = r_path.trim_end_matches('/');
+20 -9
View File
@@ -1,16 +1,27 @@
use crate::cache::create_readonly_conn;
use crate::commands::shared::{find_file_by_name, resolve_remote_file};
use crate::cache::{create_authenticated_conn, load_cache_with_token_refresh};
use crate::commands::shared::{
find_file_by_name, find_folder, find_lab_in_cache, parse_remote_path,
};
use anyhow::anyhow;
pub async fn file_metadata(remote_path: &str, password: Option<&str>) -> Result<(), anyhow::Error> {
let remote = remote_path
.splitn(2, ':')
.next()
.ok_or_else(|| anyhow!("Invalid remote path"))?;
let (conn, cache) = create_readonly_conn(remote).await?;
let (parent_folder, basename) =
resolve_remote_file(&conn, cache.as_ref(), remote_path, password).await?;
let (remote, labname, r_path) = parse_remote_path(remote_path)?;
let cache = load_cache_with_token_refresh(&remote).await?;
let conn = create_authenticated_conn(&remote, &cache)?;
let lab = find_lab_in_cache(&cache, &labname)?;
let lab_id = lab.id;
// Split the file path into parent directory and filename
let path = r_path.trim_end_matches('/');
let (dirname, basename) = if let Some(pos) = path.rfind('/') {
let d = if pos == 0 { "/" } else { &path[..pos] };
(d.to_string(), path[pos + 1..].to_string())
} else {
("/".to_string(), path.to_string())
};
let parent_folder = find_folder(&conn, lab_id, &dirname, password).await?;
let files = conn.list_all_files(&parent_folder.id).await?;
let file = find_file_by_name(&files, &basename)
+3 -2
View File
@@ -1,7 +1,8 @@
use crate::cache::create_readonly_conn;
use crate::cache::{create_authenticated_conn, load_cache_with_token_refresh};
pub async fn labs(remote: &str) -> Result<(), anyhow::Error> {
let (conn, _) = create_readonly_conn(remote).await?;
let cache = load_cache_with_token_refresh(remote).await?;
let conn = create_authenticated_conn(remote, &cache)?;
let labs = conn.list_laboratories().await?;
let header = ("Name", "PI", "Laboratory");
+9 -11
View File
@@ -1,5 +1,5 @@
use crate::cache::create_readonly_conn;
use crate::commands::shared::{fmt_datetime, resolve_remote_folder};
use crate::cache::{create_authenticated_conn, load_cache_with_token_refresh};
use crate::commands::shared::{find_folder, find_lab_in_cache, fmt_datetime, parse_remote_path};
use crate::connection::MDRSConnection;
use crate::models::file::File;
use crate::models::folder::{FolderDetail, FolderSimple};
@@ -14,14 +14,12 @@ pub async fn ls(
is_recursive: bool,
is_quiet: bool,
) -> Result<(), anyhow::Error> {
let remote = remote_path
.splitn(2, ':')
.next()
.ok_or_else(|| anyhow::anyhow!("Invalid remote path"))?;
let (conn, cache) = create_readonly_conn(remote).await?;
let (folder, lab) =
resolve_remote_folder(&conn, cache.as_ref(), None, remote_path, password).await?;
let labname = lab.name;
let (remote, labname, path) = parse_remote_path(remote_path)?;
let cache = load_cache_with_token_refresh(&remote).await?;
let conn = create_authenticated_conn(&remote, &cache)?;
let lab = find_lab_in_cache(&cache, &labname)?;
let folder = find_folder(&conn, lab.id, &path, password).await?;
if is_json {
let output = if is_recursive {
@@ -31,7 +29,7 @@ pub async fn ls(
};
println!("{}", serde_json::to_string(&output)?);
} else if is_recursive {
let prefix = format!("{}:/{}", conn.remote.as_deref().unwrap_or(""), labname);
let prefix = format!("{}:/{}", remote, labname);
ls_plain_recursive(&conn, folder, &labname, &prefix, password).await?;
} else {
let files = conn.list_all_files(&folder.id).await?;
+7 -9
View File
@@ -1,14 +1,12 @@
use crate::cache::create_readonly_conn;
use crate::commands::shared::resolve_remote_folder;
use crate::cache::{create_authenticated_conn, load_cache_with_token_refresh};
use crate::commands::shared::{find_folder, find_lab_in_cache, parse_remote_path};
pub async fn metadata(remote_path: &str, password: Option<&str>) -> Result<(), anyhow::Error> {
let remote = remote_path
.splitn(2, ':')
.next()
.ok_or_else(|| anyhow::anyhow!("Invalid remote path"))?;
let (conn, cache) = create_readonly_conn(remote).await?;
let (folder, _) =
resolve_remote_folder(&conn, cache.as_ref(), None, remote_path, password).await?;
let (remote, labname, folder_path) = parse_remote_path(remote_path)?;
let cache = load_cache_with_token_refresh(&remote).await?;
let conn = create_authenticated_conn(&remote, &cache)?;
let lab = find_lab_in_cache(&cache, &labname)?;
let folder = find_folder(&conn, lab.id, &folder_path, password).await?;
let resp = conn
.get(&format!("v3/folders/{}/metadata/", folder.id))
-395
View File
@@ -3,156 +3,9 @@ use crate::connection::ApiRequestLimiter;
use crate::connection::MDRSConnection;
use crate::models::file::File;
use crate::models::folder::{FolderDetail, FolderSimple};
use crate::models::laboratory::Laboratory;
use anyhow::{anyhow, bail};
use unicode_normalization::UnicodeNormalization;
// ---------------------------------------------------------------------------
// DOI helpers
// ---------------------------------------------------------------------------
/// Return true if the path component (after the remote: prefix) looks like a
/// DOI string, i.e. starts with "10." and contains a "/".
pub fn is_doi(path: &str) -> bool {
path.starts_with("10.") && path.contains('/')
}
/// Extract the DOI suffix ID from a full DOI string.
///
/// MDRS uses the segment after the last `.` in the DOI as its internal
/// identifier, e.g. `10.xxxx/prefix.20230511-001` → `20230511-001`.
/// If there is no `.` after the `/`, the entire suffix after `/` is used.
pub fn doi_suffix_id(doi: &str) -> &str {
// Find the slash that separates DOI prefix from suffix.
if let Some(slash_pos) = doi.find('/') {
let suffix = &doi[slash_pos + 1..];
// Use the part after the last `.` within the suffix.
if let Some(dot_pos) = suffix.rfind('.') {
&suffix[dot_pos + 1..]
} else {
suffix
}
} else {
doi
}
}
/// Split a DOI-with-optional-path string into `(doi, subpath)`.
///
/// A DOI has the form `10.REGISTRANT/SUFFIX` where SUFFIX contains no `/`.
/// Anything after the first `/` following SUFFIX is treated as a subfolder path.
///
/// Examples:
/// - `"10.1234/prefix.ID"` → `("10.1234/prefix.ID", "")`
/// - `"10.1234/prefix.ID/"` → `("10.1234/prefix.ID", "")`
/// - `"10.1234/prefix.ID/sub"` → `("10.1234/prefix.ID", "/sub")`
/// - `"10.1234/prefix.ID/sub/deep"` → `("10.1234/prefix.ID", "/sub/deep")`
pub fn split_doi_and_subpath(doi_with_path: &str) -> (&str, &str) {
// Find the first `/` that separates registrant from suffix.
if let Some(first_slash) = doi_with_path.find('/') {
let after_suffix_start = first_slash + 1;
let after_first = &doi_with_path[after_suffix_start..];
// Find the next `/` inside the suffix portion — this starts the subpath.
if let Some(second_slash) = after_first.find('/') {
let doi_end = after_suffix_start + second_slash;
let doi = &doi_with_path[..doi_end];
let subpath = &doi_with_path[doi_end..]; // begins with "/"
// Treat a bare trailing slash as no subpath (root of DOI folder).
if subpath == "/" {
(doi, "")
} else {
(doi, subpath)
}
} else {
// No second slash — the whole string is the DOI, no subpath.
(doi_with_path, "")
}
} else {
(doi_with_path, "")
}
}
/// Parse `"remote:10.xxxx/prefix.ID[/optional/sub/path]"` into
/// `(remote, doi, subpath)`.
///
/// `subpath` is empty when the remote path points directly at the DOI root
/// folder. Otherwise it is an absolute path string like `"/subfolder/deep"`.
pub fn parse_doi_remote_path(remote_path: &str) -> Result<(String, String, String), anyhow::Error> {
let parts: Vec<&str> = remote_path.splitn(2, ':').collect();
if parts.len() != 2 {
bail!("remote_path must be in the form 'remote:10.xxxx/prefix.ID'");
}
let remote = parts[0].to_string();
let doi_with_path = parts[1];
if !is_doi(doi_with_path) {
bail!(
"Path `{}` does not look like a DOI (must start with '10.' and contain '/').",
doi_with_path
);
}
let (doi, subpath) = split_doi_and_subpath(doi_with_path);
Ok((remote, doi.to_string(), subpath.to_string()))
}
/// Resolve a DOI string to a (FolderDetail, Laboratory) pair.
///
/// Calls GET v3/doi/{id}/ to look up the folder ID, then fetches the full
/// folder detail (which carries `laboratory_id`) and resolves the laboratory.
pub async fn find_folder_by_doi(
conn: &MDRSConnection,
doi: &str,
password: Option<&str>,
) -> Result<(FolderDetail, Laboratory), anyhow::Error> {
// Strip any trailing slash from the DOI before extracting the suffix ID.
let doi_clean = doi.trim_end_matches('/');
let id = doi_suffix_id(doi_clean);
let doi_resp = conn.retrieve_doi(id).await?;
// Verify that the returned DOI matches the one supplied by the caller
// (case-insensitive, trimming trailing slashes).
let returned = doi_resp.doi.trim_end_matches('/');
if !returned.eq_ignore_ascii_case(doi_clean) {
bail!(
"DOI mismatch: requested `{}` but server returned `{}`.",
doi_clean,
returned
);
}
let folder_id = &doi_resp.folder.id;
// Fetch the full folder detail; laboratory_id is available here.
let folder = conn
.retrieve_folder(folder_id)
.await
.map_err(|e| anyhow!("Failed to retrieve folder for DOI `{}`: {}", doi_clean, e))?;
// Handle password-locked folder.
if folder.lock {
match password {
None => {
bail!(
"Folder for DOI `{}` is locked. Use -p/--password to provide a password.",
doi_clean
);
}
Some(pw) => conn.folder_auth(folder_id, pw).await?,
}
}
// Resolve laboratory using the laboratory_id from the folder detail.
let lab_id = folder.laboratory_id;
let lab = conn
.list_laboratories()
.await?
.items
.into_iter()
.find(|l| l.id == lab_id)
.ok_or_else(|| anyhow!("Laboratory with id {} not found.", lab_id))?;
Ok((folder, lab))
}
// ---------------------------------------------------------------------------
// Path helpers
// ---------------------------------------------------------------------------
@@ -195,32 +48,6 @@ pub fn find_lab_in_cache<'a>(
.ok_or_else(|| anyhow!("Laboratory `{}` not found.", labname))
}
/// Resolve a laboratory by name using cached laboratories when available, and
/// falling back to the API when the user is anonymous.
pub async fn find_laboratory(
conn: &MDRSConnection,
cache: Option<&Cache>,
labname: &str,
) -> Result<Laboratory, anyhow::Error> {
if let Some(cache) = cache {
if let Ok(lab) = find_lab_in_cache(cache, labname) {
return Ok(Laboratory {
id: lab.id,
name: lab.name.clone(),
pi_name: lab.pi_name.clone(),
full_name: lab.full_name.clone(),
});
}
}
conn.list_laboratories()
.await?
.items
.into_iter()
.find(|lab| lab.name == labname)
.ok_or_else(|| anyhow!("Laboratory `{}` not found.", labname))
}
// ---------------------------------------------------------------------------
// Unicode helpers
// ---------------------------------------------------------------------------
@@ -351,225 +178,3 @@ pub fn fmt_datetime(iso: &str) -> String {
iso.to_string()
}
}
/// Resolve any remote path (normal or DOI-based) into a FolderDetail and Laboratory.
/// Takes an optional API limiter for download command compatibility.
pub async fn resolve_remote_folder(
conn: &MDRSConnection,
cache: Option<&Cache>,
limiter: Option<&ApiRequestLimiter>,
remote_path: &str,
password: Option<&str>,
) -> Result<(FolderDetail, Laboratory), anyhow::Error> {
let path_component = remote_path.splitn(2, ':').nth(1).unwrap_or("");
if is_doi(path_component) {
let (_, doi, subpath) = parse_doi_remote_path(remote_path)?;
let (doi_folder, lab) = find_folder_by_doi(conn, &doi, password).await?;
if subpath.is_empty() {
Ok((doi_folder, lab))
} else {
let abs_path = format!("{}{}", doi_folder.path.trim_end_matches('/'), subpath);
let folder = if let Some(l) = limiter {
find_folder_limited(conn, l, lab.id, &abs_path, password).await?
} else {
find_folder(conn, lab.id, &abs_path, password).await?
};
Ok((folder, lab))
}
} else {
let (_, labname, folder_path) = parse_remote_path(remote_path)?;
let lab = find_laboratory(conn, cache, &labname).await?;
let folder = if let Some(l) = limiter {
find_folder_limited(conn, l, lab.id, &folder_path, password).await?
} else {
find_folder(conn, lab.id, &folder_path, password).await?
};
Ok((folder, lab))
}
}
/// Resolves a remote path pointing to a file into the parent FolderDetail and the file's basename.
pub async fn resolve_remote_file(
conn: &MDRSConnection,
cache: Option<&Cache>,
remote_path: &str,
password: Option<&str>,
) -> Result<(FolderDetail, String), anyhow::Error> {
let path_component = remote_path.splitn(2, ':').nth(1).unwrap_or("");
if is_doi(path_component) {
let (_, doi, subpath) = parse_doi_remote_path(remote_path)?;
let (doi_folder, lab) = find_folder_by_doi(conn, &doi, password).await?;
let subpath_clean = subpath.trim_end_matches('/');
if subpath_clean.is_empty() {
bail!("DOI path must point to a file, not a folder.");
}
let (sub_dir, basename) = if let Some(pos) = subpath_clean.rfind('/') {
let d = if pos == 0 { "/" } else { &subpath_clean[..pos] };
(d.to_string(), subpath_clean[pos + 1..].to_string())
} else {
("/".to_string(), subpath_clean.to_string())
};
let abs_path = format!(
"{}{}",
doi_folder.path.trim_end_matches('/'),
if sub_dir.starts_with('/') {
sub_dir
} else {
format!("/{}", sub_dir)
}
);
let parent = find_folder(conn, lab.id, &abs_path, password).await?;
Ok((parent, basename))
} else {
let (_, labname, r_path) = parse_remote_path(remote_path)?;
let lab = find_laboratory(conn, cache, &labname).await?;
let path = r_path.trim_end_matches('/');
let (dirname, basename) = if let Some(pos) = path.rfind('/') {
let d = if pos == 0 { "/" } else { &path[..pos] };
(d.to_string(), path[pos + 1..].to_string())
} else {
("/".to_string(), path.to_string())
};
let parent = find_folder(conn, lab.id, &dirname, password).await?;
Ok((parent, basename))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[tokio::test]
async fn find_laboratory_falls_back_to_api_without_authorization() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let mut buf = [0u8; 4096];
let n = stream.read(&mut buf).await.unwrap();
let req = String::from_utf8_lossy(&buf[..n]);
assert!(req.starts_with("GET /v3/laboratories/ HTTP/1.1"));
assert!(!req.contains("\r\nAuthorization: Bearer "));
let body =
r#"[{"id":1,"name":"public-lab","pi_name":"PI","full_name":"Public Laboratory"}]"#;
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
stream.write_all(response.as_bytes()).await.unwrap();
});
let conn = MDRSConnection::new(&format!("http://{addr}"));
let lab = find_laboratory(&conn, None, "public-lab").await.unwrap();
assert_eq!(lab.id, 1);
assert_eq!(lab.name, "public-lab");
server.await.unwrap();
}
// ------------------------------------------------------------------
// DOI helper unit tests
// ------------------------------------------------------------------
#[test]
fn is_doi_returns_true_for_valid_doi() {
assert!(is_doi("10.12345/prefix.20230511-001"));
assert!(is_doi("10.1234/abc"));
}
#[test]
fn is_doi_returns_false_for_normal_paths() {
assert!(!is_doi("/labname/path/to/folder"));
assert!(!is_doi("labname/path"));
assert!(!is_doi("10.1234")); // no slash
}
#[test]
fn doi_suffix_id_extracts_last_dot_segment() {
assert_eq!(
doi_suffix_id("10.12345/prefix.20230511-001"),
"20230511-001"
);
}
#[test]
fn doi_suffix_id_no_dot_in_suffix_returns_whole_suffix() {
assert_eq!(doi_suffix_id("10.1234/nodot"), "nodot");
}
#[test]
fn doi_suffix_id_no_slash_returns_whole_input() {
assert_eq!(doi_suffix_id("10.1234"), "10.1234");
}
#[test]
fn parse_doi_remote_path_valid_no_subpath() {
let (remote, doi, subpath) =
parse_doi_remote_path("neurodata:10.12345/prefix.20230511-001").unwrap();
assert_eq!(remote, "neurodata");
assert_eq!(doi, "10.12345/prefix.20230511-001");
assert_eq!(subpath, "");
}
#[test]
fn parse_doi_remote_path_valid_trailing_slash() {
let (remote, doi, subpath) =
parse_doi_remote_path("neurodata:10.12345/prefix.20230511-001/").unwrap();
assert_eq!(remote, "neurodata");
assert_eq!(doi, "10.12345/prefix.20230511-001");
assert_eq!(subpath, ""); // trailing slash treated as no subpath
}
#[test]
fn parse_doi_remote_path_valid_with_subpath() {
let (remote, doi, subpath) =
parse_doi_remote_path("neurodata:10.12345/prefix.20230511-001/sub/folder").unwrap();
assert_eq!(remote, "neurodata");
assert_eq!(doi, "10.12345/prefix.20230511-001");
assert_eq!(subpath, "/sub/folder");
}
#[test]
fn parse_doi_remote_path_rejects_normal_path() {
let err = parse_doi_remote_path("neurodata:/lab/path").unwrap_err();
assert!(err.to_string().contains("does not look like a DOI"));
}
#[test]
fn split_doi_and_subpath_no_subpath() {
assert_eq!(
split_doi_and_subpath("10.1234/prefix.ID"),
("10.1234/prefix.ID", "")
);
}
#[test]
fn split_doi_and_subpath_trailing_slash_only() {
assert_eq!(
split_doi_and_subpath("10.1234/prefix.ID/"),
("10.1234/prefix.ID", "")
);
}
#[test]
fn split_doi_and_subpath_single_level() {
assert_eq!(
split_doi_and_subpath("10.1234/prefix.ID/sub"),
("10.1234/prefix.ID", "/sub")
);
}
#[test]
fn split_doi_and_subpath_multi_level() {
assert_eq!(
split_doi_and_subpath("10.1234/prefix.ID/sub/deep/path"),
("10.1234/prefix.ID", "/sub/deep/path")
);
}
}
-18
View File
@@ -1,18 +0,0 @@
use serde::{Deserialize, Serialize};
/// Nested folder information returned inside a DOI response.
/// The DOI endpoint only returns the folder `id`; `laboratory_id` must be
/// obtained by subsequently calling the folder retrieve endpoint.
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DoiFolderRef {
pub id: String,
}
/// Response from GET v3/doi/{id}/
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DoiResponse {
/// The internal DOI suffix ID (e.g. "20260429-001") returned as a string.
pub id: String,
pub doi: String,
pub folder: DoiFolderRef,
}
-1
View File
@@ -1,6 +1,5 @@
// Model definitions (User, Laboratory, File, Folder, etc.)
pub mod doi;
pub mod file;
pub mod folder;
pub mod laboratory;