Files
mdrs-client-rust/src/cache/mod.rs
T
Yoshihiro OKUMURA a16f73543d fix: stream transfers and close gaps against the Python client
A review against the Python client, which shares this client's config
and login cache, found the transfers holding whole files in memory and
several commands answering differently from their Python counterparts.

- Stream uploads and downloads instead of buffering the whole file, so
  memory no longer scales with file size times concurrency. Uploads
  still declare a Content-Length rather than going out chunked.
- Write a download beside its destination and move it into place once
  complete, and refuse a destination that cannot be written, so a
  failed transfer leaves what was already there untouched.
- Report the server's own error detail instead of the bare status.
- List locked sub-folders in `ls --json --recursive` using the given
  password and skip only those that cannot be unlocked, instead of
  failing the whole listing.
- Name each folder's laboratory in JSON output and sort its entries,
  matching the Python client's schema and order.
- Ask the server who is logged in for `whoami`, rather than trusting
  the cached name.
- Collapse repeated separators and refuse `..` in remote paths.
- Fall back to the API when a laboratory is missing from the cache, so
  a newly added one no longer needs a fresh login.
- Verify the login cache digest on read, as the Python client does.
- Accept `-e` as the short form of `--exclude`.
- Satisfy clippy and rustfmt across the crate.
2026-09-04 16:31:14 +09:00

671 lines
23 KiB
Rust

pub mod digest;
pub mod types;
pub use digest::compute_digest;
pub use types::{Cache, CacheLaboratory, CacheLabsWrapper, CacheToken, CacheUser};
use crate::connection::MDRSConnection;
use anyhow::{anyhow, bail};
use std::collections::HashMap;
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::{MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex};
use std::time::UNIX_EPOCH;
// ---------------------------------------------------------------------------
// Per-remote async mutex map (in-process serialization)
// ---------------------------------------------------------------------------
static REMOTE_LOCKS: LazyLock<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct CacheStoreKey {
config_dir: PathBuf,
remote: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct CacheFileSnapshot {
len: u64,
modified_nanos: u128,
#[cfg(unix)]
inode: u64,
}
#[derive(Clone)]
struct MemoryCacheEntry {
snapshot: CacheFileSnapshot,
cache: Cache,
}
static MEMORY_CACHE: LazyLock<Mutex<HashMap<CacheStoreKey, MemoryCacheEntry>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
fn get_remote_lock(remote: &str) -> Arc<tokio::sync::Mutex<()>> {
let mut map = REMOTE_LOCKS.lock().unwrap();
map.entry(remote.to_string())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
// ---------------------------------------------------------------------------
// Cache file path helpers
// ---------------------------------------------------------------------------
fn cache_store_key(config_dir: &Path, remote: &str) -> CacheStoreKey {
CacheStoreKey {
config_dir: config_dir.to_path_buf(),
remote: remote.to_string(),
}
}
fn cache_dir_path(config_dir: &Path) -> PathBuf {
config_dir.join("cache")
}
fn cache_file_path_in(config_dir: &Path, remote: &str) -> PathBuf {
cache_dir_path(config_dir).join(format!("{}.json", remote))
}
fn cache_file_path(remote: &str) -> PathBuf {
cache_file_path_in(&crate::settings::SETTINGS.config_dirname, remote)
}
fn cache_snapshot(metadata: &fs::Metadata) -> CacheFileSnapshot {
let modified_nanos = metadata
.modified()
.ok()
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.map(|duration| duration.as_nanos())
.unwrap_or_default();
CacheFileSnapshot {
len: metadata.len(),
modified_nanos,
#[cfg(unix)]
inode: metadata.ino(),
}
}
fn read_cache_snapshot(cache_path: &Path) -> Result<CacheFileSnapshot, std::io::Error> {
fs::metadata(cache_path).map(|metadata| cache_snapshot(&metadata))
}
fn cached_entry(config_dir: &Path, remote: &str, snapshot: &CacheFileSnapshot) -> Option<Cache> {
let key = cache_store_key(config_dir, remote);
let map = MEMORY_CACHE.lock().unwrap();
map.get(&key)
.filter(|entry| entry.snapshot == *snapshot)
.map(|entry| entry.cache.clone())
}
fn update_cached_entry(config_dir: &Path, remote: &str, snapshot: CacheFileSnapshot, cache: Cache) {
let key = cache_store_key(config_dir, remote);
let mut map = MEMORY_CACHE.lock().unwrap();
map.insert(key, MemoryCacheEntry { snapshot, cache });
}
fn invalidate_cached_entry(config_dir: &Path, remote: &str) {
let key = cache_store_key(config_dir, remote);
let mut map = MEMORY_CACHE.lock().unwrap();
map.remove(&key);
}
fn ensure_cache_dir(cache_dir: &Path) -> Result<(), anyhow::Error> {
fs::create_dir_all(cache_dir)?;
#[cfg(unix)]
{
let mut perms = fs::metadata(cache_dir)?.permissions();
perms.set_mode(0o700);
fs::set_permissions(cache_dir, perms)?;
}
Ok(())
}
fn write_cache_file(cache_path: &Path, cache: &Cache) -> Result<(), anyhow::Error> {
let tmp_path = cache_path.with_extension("tmp");
fs::write(&tmp_path, serde_json::to_vec_pretty(cache)?)?;
#[cfg(unix)]
{
let mut perms = fs::metadata(&tmp_path)?.permissions();
perms.set_mode(0o600);
fs::set_permissions(&tmp_path, perms)?;
}
fs::rename(&tmp_path, cache_path)?;
Ok(())
}
fn parse_cache(remote: &str, data: &str) -> Result<Cache, anyhow::Error> {
let cache = serde_json::from_str::<Cache>(data).map_err(|e| {
anyhow!(
"Cache for `{}` is invalid or outdated ({}). Run `mdrs login {}` to refresh it.",
remote,
e,
remote
)
})?;
// The digest is what says the file is still the one this client wrote. The Python
// client checks it on every read and the two share the file, so a cache one of them
// would refuse must not be honoured by the other.
let expected = compute_digest(
cache.user.as_ref(),
&cache.token.access,
&cache.token.refresh,
&cache.laboratories,
);
if cache.digest != expected {
bail!(
"Cache for `{}` has been altered or was written by an incompatible version. \
Run `mdrs login {}` to refresh it.",
remote,
remote
);
}
Ok(cache)
}
fn load_cache_from_dir(remote: &str, config_dir: &Path) -> Result<Cache, anyhow::Error> {
let cache_path = cache_file_path_in(config_dir, remote);
let snapshot = match read_cache_snapshot(&cache_path) {
Ok(snapshot) => snapshot,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
bail!(
"Not logged in to `{}`. Run `mdrs login {}` first.",
remote,
remote
);
}
Err(e) => return Err(e.into()),
};
if let Some(cache) = cached_entry(config_dir, remote, &snapshot) {
return Ok(cache);
}
let data = match fs::read_to_string(&cache_path) {
Ok(data) => data,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
bail!(
"Not logged in to `{}`. Run `mdrs login {}` first.",
remote,
remote
);
}
Err(e) => return Err(e.into()),
};
let cache = parse_cache(remote, &data)?;
update_cached_entry(config_dir, remote, snapshot, cache.clone());
Ok(cache)
}
fn load_cache_if_present_from_dir(
remote: &str,
config_dir: &Path,
) -> Result<Option<Cache>, anyhow::Error> {
let cache_path = cache_file_path_in(config_dir, remote);
let snapshot = match read_cache_snapshot(&cache_path) {
Ok(snapshot) => snapshot,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
return Ok(None);
}
Err(e) => return Err(e.into()),
};
if let Some(cache) = cached_entry(config_dir, remote, &snapshot) {
return Ok(Some(cache));
}
let data = match fs::read_to_string(&cache_path) {
Ok(data) => data,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
invalidate_cached_entry(config_dir, remote);
return Ok(None);
}
Err(e) => return Err(e.into()),
};
let cache = match parse_cache(remote, &data) {
Ok(cache) => cache,
Err(e) => {
// Said out loud before the credentials go. The alternative is a command that
// quietly carries on anonymously and fails later for a reason that looks
// unrelated to the cache it just threw away.
eprintln!("{e}");
remove_cache_in_dir(remote, config_dir)?;
return Ok(None);
}
};
update_cached_entry(config_dir, remote, snapshot, cache.clone());
Ok(Some(cache))
}
fn persist_cache_in_dir(
remote: &str,
config_dir: &Path,
cache: &Cache,
) -> Result<(), anyhow::Error> {
let cache_dir = cache_dir_path(config_dir);
ensure_cache_dir(&cache_dir)?;
let cache_path = cache_file_path_in(config_dir, remote);
write_cache_file(&cache_path, cache)?;
let snapshot = read_cache_snapshot(&cache_path)?;
update_cached_entry(config_dir, remote, snapshot, cache.clone());
Ok(())
}
fn remove_cache_in_dir(remote: &str, config_dir: &Path) -> Result<(), anyhow::Error> {
let cache_path = cache_file_path_in(config_dir, remote);
match fs::remove_file(&cache_path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
invalidate_cached_entry(config_dir, remote);
Ok(())
}
// ---------------------------------------------------------------------------
// Load cache (low-level, no token refresh)
// ---------------------------------------------------------------------------
/// Load token and laboratories from the login cache file (no token refresh check).
pub fn load_cache(remote: &str) -> Result<Cache, anyhow::Error> {
load_cache_from_dir(remote, &crate::settings::SETTINGS.config_dirname)
}
/// Persist a cache entry and refresh the in-memory fast path.
pub fn persist_cache(remote: &str, cache: &Cache) -> Result<(), anyhow::Error> {
persist_cache_in_dir(remote, &crate::settings::SETTINGS.config_dirname, cache)
}
/// Remove a cache entry from disk and memory.
pub fn remove_cache(remote: &str) -> Result<(), anyhow::Error> {
remove_cache_in_dir(remote, &crate::settings::SETTINGS.config_dirname)
}
// ---------------------------------------------------------------------------
// Token-aware cache load with refresh and locking
// ---------------------------------------------------------------------------
/// Load cache, check token expiry, and refresh the access token if needed.
///
/// Locking strategy:
/// - Per-remote `tokio::sync::Mutex` serializes concurrent async tasks within
/// the same process.
/// - `flock(LOCK_EX)` on a dedicated `cache/{remote}.lock` file serializes
/// the entire read-check-refresh-write cycle across separate processes on
/// the same host.
pub async fn load_cache_with_token_refresh(remote: &str) -> Result<Cache, anyhow::Error> {
let lock = get_remote_lock(remote);
let _guard = lock.lock().await;
ensure_cache_dir(&cache_dir_path(&crate::settings::SETTINGS.config_dirname))?;
let lock_path = cache_file_path(remote).with_extension("lock");
use fs2::FileExt;
let lock_file = fs::OpenOptions::new()
.write(true)
.create(true)
// Nothing is ever written into it: the file exists only to be flocked, and
// truncating it would touch a file other processes are holding open.
.truncate(false)
.open(&lock_path)?;
lock_file.lock_exclusive()?;
// Re-read inside the lock: another process may have already refreshed the
// token since we last checked.
let result: Result<Cache, anyhow::Error> = async {
let mut cache = load_cache(remote)?;
if crate::token::is_expired(&cache.token.refresh) {
bail!(
"Session for `{}` has expired. Please run `mdrs login {}` again.",
remote,
remote
);
}
if crate::token::is_refresh_required(&cache.token.access, &cache.token.refresh) {
cache = refresh_and_persist(remote, &cache).await?;
}
Ok(cache)
}
.await;
lock_file.unlock()?;
result
}
async fn load_cache_with_token_refresh_optional_from_dir(
remote: &str,
config_dir: &Path,
) -> Result<Option<Cache>, anyhow::Error> {
let lock = get_remote_lock(remote);
let _guard = lock.lock().await;
ensure_cache_dir(&cache_dir_path(config_dir))?;
let lock_path = cache_file_path_in(config_dir, remote).with_extension("lock");
use fs2::FileExt;
let lock_file = fs::OpenOptions::new()
.write(true)
.create(true)
// Nothing is ever written into it: the file exists only to be flocked, and
// truncating it would touch a file other processes are holding open.
.truncate(false)
.open(&lock_path)?;
lock_file.lock_exclusive()?;
let result: Result<Option<Cache>, anyhow::Error> = async {
let Some(mut cache) = load_cache_if_present_from_dir(remote, config_dir)? else {
return Ok(None);
};
if crate::token::is_expired(&cache.token.refresh) {
remove_cache_in_dir(remote, config_dir)?;
return Ok(None);
}
if crate::token::is_refresh_required(&cache.token.access, &cache.token.refresh) {
cache = refresh_and_persist_in_dir(remote, config_dir, &cache).await?;
}
Ok(Some(cache))
}
.await;
lock_file.unlock()?;
result
}
/// Load cache when present and refresh its token if needed.
///
/// Unlike `load_cache_with_token_refresh`, this returns `Ok(None)` when the user
/// is effectively anonymous: no cache file exists, the cache is invalid, or the
/// refresh token has already expired. This mirrors the Python client behavior
/// used by read-only commands.
pub async fn load_cache_with_token_refresh_optional(
remote: &str,
) -> Result<Option<Cache>, anyhow::Error> {
load_cache_with_token_refresh_optional_from_dir(
remote,
&crate::settings::SETTINGS.config_dirname,
)
.await
}
/// Call the token-refresh endpoint and write the new access token back to the
/// cache file. The caller must already hold the per-remote async mutex.
async fn refresh_and_persist(remote: &str, cache: &Cache) -> Result<Cache, anyhow::Error> {
refresh_and_persist_in_dir(remote, &crate::settings::SETTINGS.config_dirname, cache).await
}
async fn refresh_and_persist_in_dir(
remote: &str,
config_dir: &Path,
cache: &Cache,
) -> Result<Cache, anyhow::Error> {
let url = crate::commands::config::get_remote_url(remote)?
.ok_or_else(|| anyhow!("Remote `{}` is not configured.", remote))?;
let conn = MDRSConnection::new(&url);
let refreshed = conn.token_refresh(&cache.token.refresh).await?;
let mut updated_cache = cache.clone();
updated_cache.token.access = refreshed.access;
if let Some(refresh) = refreshed.refresh {
updated_cache.token.refresh = refresh;
}
updated_cache.digest = compute_digest(
updated_cache.user.as_ref(),
&updated_cache.token.access,
&updated_cache.token.refresh,
&updated_cache.laboratories,
);
persist_cache_in_dir(remote, config_dir, &updated_cache)?;
Ok(updated_cache)
}
// ---------------------------------------------------------------------------
// Connection helpers
// ---------------------------------------------------------------------------
/// Create an authenticated `MDRSConnection` for the given remote label.
pub fn create_authenticated_conn(
remote: &str,
cache: &Cache,
) -> Result<MDRSConnection, anyhow::Error> {
Ok(create_remote_conn(remote)?.with_token(cache.token.access.clone()))
}
/// Create an unauthenticated `MDRSConnection` for the given remote label.
pub fn create_remote_conn(remote: &str) -> Result<MDRSConnection, anyhow::Error> {
let url = crate::commands::config::get_remote_url(remote)?
.ok_or_else(|| anyhow!("Remote `{}` is not configured.", remote))?;
Ok(MDRSConnection::new(&url).with_remote(remote))
}
/// Create a connection for read-only commands, attaching a bearer token only
/// when a valid login cache is available.
pub async fn create_readonly_conn(
remote: &str,
) -> Result<(MDRSConnection, Option<Cache>), anyhow::Error> {
let conn = create_remote_conn(remote)?;
match load_cache_with_token_refresh_optional(remote).await? {
Some(cache) => Ok((conn.with_token(cache.token.access.clone()), Some(cache))),
None => Ok((conn, None)),
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
fn sample_cache(username: &str) -> Cache {
let mut cache = Cache {
user: Some(CacheUser {
id: 1,
username: username.to_string(),
laboratory_ids: vec![10, 20],
is_reviewer: false,
}),
token: types::CacheToken {
access: format!("access-{username}"),
refresh: format!("refresh-{username}"),
},
laboratories: CacheLabsWrapper {
items: vec![CacheLaboratory {
id: 10,
name: "lab".to_string(),
pi_name: String::new(),
full_name: "Laboratory".to_string(),
}],
},
digest: String::new(),
};
cache.digest = compute_digest(
cache.user.as_ref(),
&cache.token.access,
&cache.token.refresh,
&cache.laboratories,
);
cache
}
fn remote_name(prefix: &str, config_dir: &Path) -> String {
format!(
"{prefix}-{}",
config_dir
.file_name()
.unwrap_or_default()
.to_string_lossy()
.replace('.', "_")
)
}
#[cfg(unix)]
#[test]
fn load_cache_uses_memory_fast_path_when_snapshot_matches() {
let dir = tempdir().unwrap();
let remote = remote_name("fast-path", dir.path());
let cache = sample_cache("alice");
persist_cache_in_dir(&remote, dir.path(), &cache).unwrap();
let first = load_cache_from_dir(&remote, dir.path()).unwrap();
assert_eq!(first.user.unwrap().username, "alice");
let cache_path = cache_file_path_in(dir.path(), &remote);
let mut perms = fs::metadata(&cache_path).unwrap().permissions();
perms.set_mode(0o000);
fs::set_permissions(&cache_path, perms).unwrap();
let second = load_cache_from_dir(&remote, dir.path()).unwrap();
assert_eq!(second.user.unwrap().username, "alice");
}
#[test]
fn load_cache_reloads_when_external_writer_changes_file() {
let dir = tempdir().unwrap();
let remote = remote_name("reload", dir.path());
let original = sample_cache("alice");
let updated = sample_cache("bob");
persist_cache_in_dir(&remote, dir.path(), &original).unwrap();
let first = load_cache_from_dir(&remote, dir.path()).unwrap();
assert_eq!(first.user.unwrap().username, "alice");
let cache_dir = cache_dir_path(dir.path());
ensure_cache_dir(&cache_dir).unwrap();
let cache_path = cache_file_path_in(dir.path(), &remote);
write_cache_file(&cache_path, &updated).unwrap();
let second = load_cache_from_dir(&remote, dir.path()).unwrap();
assert_eq!(second.user.unwrap().username, "bob");
}
#[cfg(unix)]
#[test]
fn persist_cache_refreshes_memory_entry() {
let dir = tempdir().unwrap();
let remote = remote_name("persist", dir.path());
let original = sample_cache("alice");
let updated = sample_cache("bob");
persist_cache_in_dir(&remote, dir.path(), &original).unwrap();
let _ = load_cache_from_dir(&remote, dir.path()).unwrap();
persist_cache_in_dir(&remote, dir.path(), &updated).unwrap();
let cache_path = cache_file_path_in(dir.path(), &remote);
let mut perms = fs::metadata(&cache_path).unwrap().permissions();
perms.set_mode(0o000);
fs::set_permissions(&cache_path, perms).unwrap();
let loaded = load_cache_from_dir(&remote, dir.path()).unwrap();
assert_eq!(loaded.user.unwrap().username, "bob");
}
#[test]
fn remove_cache_invalidates_memory_entry() {
let dir = tempdir().unwrap();
let remote = remote_name("remove", dir.path());
let cache = sample_cache("alice");
persist_cache_in_dir(&remote, dir.path(), &cache).unwrap();
let _ = load_cache_from_dir(&remote, dir.path()).unwrap();
remove_cache_in_dir(&remote, dir.path()).unwrap();
let err = load_cache_from_dir(&remote, dir.path()).unwrap_err();
assert!(
err.to_string()
.contains(&format!("Not logged in to `{remote}`"))
);
}
fn make_jwt_with_exp(exp: i64) -> String {
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#);
let payload = URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#));
format!("{header}.{payload}.")
}
#[test]
fn load_cache_if_present_returns_none_when_cache_missing() {
let dir = tempdir().unwrap();
let remote = remote_name("missing", dir.path());
let loaded = load_cache_if_present_from_dir(&remote, dir.path()).unwrap();
assert!(loaded.is_none());
}
/// A cache whose contents no longer match its digest is not a session to act on.
#[test]
fn load_cache_refuses_a_tampered_cache() {
let dir = tempdir().unwrap();
let remote = remote_name("tampered", dir.path());
let mut cache = sample_cache("alice");
persist_cache_in_dir(&remote, dir.path(), &cache).unwrap();
// Someone edits the stored token but leaves the digest as it was.
cache.token.access = "someone-elses-access".to_string();
let cache_path = cache_file_path_in(dir.path(), &remote);
fs::write(&cache_path, serde_json::to_vec_pretty(&cache).unwrap()).unwrap();
invalidate_cached_entry(dir.path(), &remote);
let err = load_cache_from_dir(&remote, dir.path()).unwrap_err();
assert!(err.to_string().contains("has been altered"));
}
#[test]
fn load_cache_if_present_clears_invalid_cache() {
let dir = tempdir().unwrap();
let remote = remote_name("invalid", dir.path());
let cache_dir = cache_dir_path(dir.path());
ensure_cache_dir(&cache_dir).unwrap();
let cache_path = cache_file_path_in(dir.path(), &remote);
fs::write(&cache_path, b"{invalid json").unwrap();
let loaded = load_cache_if_present_from_dir(&remote, dir.path()).unwrap();
assert!(loaded.is_none());
assert!(!cache_path.exists());
}
#[tokio::test]
async fn optional_cache_load_treats_expired_session_as_anonymous() {
let dir = tempdir().unwrap();
let remote = remote_name("expired", dir.path());
let mut cache = sample_cache("alice");
cache.token.access = make_jwt_with_exp(0);
cache.token.refresh = make_jwt_with_exp(0);
// Re-stamped, so this exercises the expired session rather than the digest check.
cache.digest = compute_digest(
cache.user.as_ref(),
&cache.token.access,
&cache.token.refresh,
&cache.laboratories,
);
persist_cache_in_dir(&remote, dir.path(), &cache).unwrap();
let loaded = load_cache_with_token_refresh_optional_from_dir(&remote, dir.path())
.await
.unwrap();
assert!(loaded.is_none());
assert!(!cache_file_path_in(dir.path(), &remote).exists());
}
}