docs: release 1.3.19 and document the environment variables

- Close the Unreleased section as 1.3.19, summarising it by what
  changed for the user rather than by what each commit touched.
- Add the environment variables the settings module reads, which the
  README did not mention.

.cspell.json

@@ -0,0 +1,26 @@
+{
+  "version": "0.2",
+  "language": "en,en-gb",
+  "ignoreWords": [
+    "followlinks",
+    "getframe",
+    "pycache",
+    "pydantic",
+    "toolbelt",
+    "UNLCK"
+  ],
+  "words": [
+    "chacl",
+    "kikan",
+    "mdrs",
+    "mdrsclient",
+    "neurodata",
+    "Neuroinformatics",
+    "orcid",
+    "RIKEN"
+  ],
+  "ignorePaths": [
+    ".env",
+    "__pycache__"
+  ]
+}

.editorconfig

@@ -0,0 +1,12 @@
+# editorconfig.org
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false

.gitignore

@@ -161,3 +161,4 @@ cython_debug/
 
 # mdrs-cli
 .neurodatacli.config
+poetry.toml

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
   "editor.formatOnSave": true,
   "editor.codeActionsOnSave": {
-    "source.organizeImports": true
+    "source.organizeImports": "explicit"
   },
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
@@ -15,9 +15,12 @@
   "[jsonc]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
+  // Extensions - Black Formatter
+  "black-formatter.args": ["--line-length=120"],
   // Extensions - Code Spell Checker
-  "cSpell.ignoreWords": ["getframe", "pydantic", "UNLCK"],
+  // - see: .cspell.json
-  "cSpell.words": ["chacl", "mdrs", "mdrsclient", "neurodata", "Neuroinformatics", "RIKEN"],
+  // Extensions - Flake8
+  "flake8.args": ["--max-line-length=120"],
   // Extensions - isort
   "isort.args": ["--profile=black"],
   // Extensions - Prettier
@@ -28,16 +31,6 @@
   "prettier.trailingComma": "all",
   // Extensions - Pylance
   "python.analysis.typeCheckingMode": "strict",
-  "python.analysis.exclude": ["api/migrations/[0-9]*.py"],
-  // Extensions - Python:black
-  "python.formatting.blackArgs": ["--line-length=120"],
-  "python.formatting.provider": "black",
-  // Extensions - Python:Flake8
-  "python.linting.enabled": true,
-  "python.linting.flake8Enabled": true,
-  "python.linting.flake8Args": ["--max-line-length=120"],
-  "python.linting.ignorePatterns": ["**/site-packages/**/*.py", ".vscode/*.py"],
-  "python.linting.lintOnSave": true,
   // Extensions - Python Docstring Generator configuration
   "autoDocstring.docstringFormat": "google"
 }

CHANGELOG.md

@@ -0,0 +1,157 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [1.3.19] - 2026-09-04
+
+### Fixed
+- Reported failed transfers instead of ending in success, so a run that lost files exits non-zero.
+- Left an existing local file intact when a download fails part way through.
+- Opened password-protected DOI folders, which previously failed outright.
+- Kept a session working across concurrent runs and long server queues.
+- Other minor fixes to `ls` output and remote URL handling.
+
+### Changed
+- Moved packaging to PEP 621 metadata with flit as the build backend.
+- Updated dependencies.
+- **Breaking for embedders:** `CacheInterface` now requires `lock()` and `reload()`.
+
+## [1.3.18] - 2026-07-02
+
+### Added
+- Added a comprehensive unit test suite in `tests/test_commands.py` checking registration, parsing, and execution flow of all 16 commands.
+
+### Refactored
+- Abstracted configuration storage (introducing `ConfigInterface`, `InMemoryConfig`, and updating `ConfigFile`) to enable dependency injection.
+- Modularized transfer operations (upload and download) to decouple them from the service layer.
+- Decoupled commands from direct file system configurations and migrated all subcommands to use abstract config classes.
+
+### Changed
+- Upgraded dependencies including `pydantic-settings` to `2.14.2` and `pyright` to `1.1.411`.
+
+### Fixed
+- Fixed duplicate `__all__` definitions in package initialization file `mdrsclient/__init__.py` that caused `__version__` export to be overwritten.
+
+## [1.3.17] - 2026-07-02
+
+### Refactored
+- Decoupled core logic from CLI interface and introduced `MdrsClient` service layer to improve library portability.
+- Migrated all CLI commands to utilize `MdrsClient` for execution.
+
+### Added
+- Abstract authentication state using `CacheInterface` and `InMemoryCache`.
+
+## [1.3.16] - 2026-06-12
+
+### Fixed
+- Retrieve the full `Folder` object from `FoldersApi` instead of using the `FolderSimple` returned by `find_sub_folder` when resolving DOI subfolders. This fixes a type checker error under the upgraded pyright and avoids a potential AttributeError at runtime due to `FolderSimple` lacking the `path` attribute.
+
+### Changed
+- Upgraded dependencies and bumped version to 1.3.16 in pyproject.toml.
+
+## [1.3.15] - 2026-05-01
+
+### Fixed
+- Apply NFC normalization to filenames and folder names sent to the server.
+
+## [1.3.14] - 2026-04-17
+
+### Changed
+- Simplified `config list` command (removed `-l`/`--long` option, always display URL).
+- Renamed `--quick` option to `--quiet` for `ls` subcommand.
+
+### Added
+- Added subcommand aliases for config commands (e.g. `ls` alias for list, `rm` alias for delete).
+- Added `version` command.
+
+## [1.3.13] - 2025-07-02
+
+### Fixed
+- Fixed pagination logic for the `file.list` API.
+
+## [1.3.12] - 2025-05-20
+
+### Fixed
+- Fixed bug where file downloading was skipped incorrectly when `-s`/`--skip-if-file-exists` option was present.
+
+## [1.3.11] - 2025-01-21
+
+### Fixed
+- Follow-up fixes for User API specification changes.
+
+## [1.3.10] - 2024-12-23
+
+### Added
+- Delete broken files and show a summary when a file download fails.
+
+### Changed
+- Updated dependency libraries.
+
+## [1.3.9] - 2024-10-23
+
+### Fixed
+- Fixed compatibility with Python 3.10.
+
+## [1.3.8] - 2024-09-18
+
+### Added
+- Implemented `-s`/`--skip-if-file-exists` option for `download` command.
+
+### Fixed
+- Added exception handling for unexpected responses from the server.
+
+## [1.3.7] - 2024-07-22
+
+### Added
+- Implemented `--exclude` argument for download subcommand.
+
+## [1.3.6] - 2024-07-08
+
+### Added
+- Support cancelling recursive downloads if downloading some files fails.
+
+## [1.3.5] - 2024-07-08
+
+### Added
+- Added authorization token validation checks for file download operations.
+
+### Removed
+- Removed unnecessary debug code.
+
+## [1.3.4] - 2024-07-04
+
+### Added
+- Added some aliases for config sub command.
+
+### Fixed
+- Fixed bug when uploading large files.
+
+## [1.3.3] - 2024-02-13
+
+### Added
+- Implemented `-s`/`--skip-if-file-exists` option for `upload` command.
+
+## [1.3.2] - 2024-02-09
+
+### Added
+- Added `-u` and `-p` options to login command.
+
+## [1.3.1] - 2023-12-20
+
+### Fixed
+- Fixed bug to resolve local files for recursive file upload.
+
+## [1.3.0] - 2023-12-18
+
+### Changed
+- Removed debug comments.
+
+## [1.2.0] - 2023-10-04
+
+### Changed
+- Follow-up recent specification changes about folder access level.
+
+## [1.1.1] - 2023-07-26
+
+### Changed
+- Set destination folder name using name attribute of folder copy API.

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS
+Copyright (c) 2023- Neuroinformatics Unit, RIKEN CBS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -2,153 +2,290 @@
 
 The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository.
 
-## Installing
+## Installation (Production)
 
-```
+It is recommended to use a virtual environment (`venv`) to keep your Python environment isolated.
-pip install -e .
+
+### Option 1: Using standard `venv` + `pip`
+
+```shell
+python3 -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+pip install .
 ```
 
-## Example Usage
+### Option 2: Using `uv`
+
+```shell
+uv venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+uv pip install .
+```
+
+## Development Setup
+
+To set up a local development environment with development tools (testing, formatting, linting):
+
+### Option 1: Using standard `venv` + `pip`
+
+```shell
+python3 -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+pip install -e ".[dev]"
+```
+
+### Option 2: Using `uv` (Recommended for developers)
+
+```shell
+uv sync
+```
+
+## Configuration
+
+The following environment variables can be set, either in the shell or via a `.env` file placed in the working directory.
+
+| Variable | Default | Description |
+|---|---|---|
+| `MDRS_CLIENT_CONFIG_DIRNAME` | `~/.mdrs-client` | Directory where config and login cache files are stored |
+| `MDRS_CLIENT_CONCURRENT` | `10` | Number of concurrent file transfers for upload and download |
+
+## CLI Usage
 
 ### config create
 
 Create remote host configuration
 
+```shell
+mdrs config create neurodata https://neurodata.riken.jp/api
 ```
-$ mdrs config create neurodata https://neurodata.riken.jp/api
+
+### config update
+
+Update the URL of a registered remote host.
+
+```shell
+mdrs config update neurodata https://neurodata.riken.jp/api
+```
+
+### config list
+
+List registered remote hosts.
+
+```shell
+mdrs config list
+mdrs config ls
+```
+
+### config delete
+
+Remove a registered remote host.
+
+```shell
+mdrs config delete neurodata
+mdrs config rm neurodata
 ```
 
 ### login
 
 Login to remote host
 
-```
+```shell
-$ mdrs login neurodata:
+mdrs login neurodata:
 Username: (enter your login name)
 Password: (enter your password)
+
+mdrs login -u USERNAME -p PASSWORD neurodata:
 ```
 
 ### logout
 
 Logout from remote host
 
-```
+```shell
-$ mdrs logout neurodata:
+mdrs logout neurodata:
 ```
 
 ### whoami
 
 Print current user name
 
-```
+```shell
-$ mdrs whoami neurodata:
+mdrs whoami neurodata:
 ```
 
 ### labs
 
 List all laboratories
 
-```
+```shell
-$ mdrs labs neurodata:
+mdrs labs neurodata:
 ```
 
 ### ls
 
-List the folder contents
+List the folder contents. You can also specify a DOI path in the form `remote:10.xxxx/yyy.ID[/optional/subpath]`.
 
-```
+```shell
-$ mdrs ls neurodata:/NIU/Repository/
+mdrs ls neurodata:/NIU/Repository/
-$ mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
+mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
-$ mdrs ls -r neurodata:/NIU/Repository/Dataset1/
+mdrs ls -r neurodata:/NIU/Repository/Dataset1/
-$ mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
+mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
+
+# DOI access examples:
+mdrs ls neurodata:10.60178/cbs.20260429-001
+mdrs ls "neurodata:10.60178/cbs.20260429-001/Figure 1"
 ```
 
 ### mkdir
 
 Create a new folder
 
-```
+```shell
-$ mdrs mkdir neurodata:/NIU/Repository/TEST
+mdrs mkdir neurodata:/NIU/Repository/TEST
 ```
 
 ### upload
 
 Upload the file or directory
 
-```
+```shell
-$ mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
+mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
-$ mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
+mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
+mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
 ```
 
 ### download
 
-Download the file or folder
+Download the file or folder. You can also specify a DOI path.
 
-```
+```shell
-$ mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
+mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
-$ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
+mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
-$ mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
+mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
+mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
+mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
+
+# DOI access examples:
+mdrs download neurodata:10.60178/cbs.20260429-001/README_NeuroData.md ./
+mdrs download -r "neurodata:10.60178/cbs.20260429-001/Figure 1" ./
 ```
 
 ### mv
 
 Move or rename the file or folder
 
-```
+```shell
-$ mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
-$ mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
+mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
 ```
 
 ### cp
 
 Copy the file and folder
 
-```
+```shell
-$ mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
-$ mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
+mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
 ```
 
 ### rm
 
 Remove the file or folder
 
-```
+```shell
-$ mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs rm neurodata:/NIU/Repository/TEST/sample.dat
-$ mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset
+mdrs rm -r neurodata:/NIU/Repository/TEST/dataset
 ```
 
 ### chacl
 
 Change the folder access level
 
-```
+```shell
-$ mdrs chacl private neurodata:/NIU/Repository/Private
+mdrs chacl private neurodata:/NIU/Repository/Private
-$ mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
+mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
-$ mdrs chacl pw_open -r -p FOLDER_PASSWORD neurodata:/NIU/Repository/PW_Open
+mdrs chacl pw_open -r -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open
 ```
 
 ### metadata
 
-Get a folder metadata
+Get a folder metadata. You can also specify a DOI path.
 
-```
+```shell
-$ mdrs metadata neurodata:/NIU/Repository/TEST/
+mdrs metadata neurodata:/NIU/Repository/TEST/
-$ mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
+mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
+
+# DOI access examples:
+mdrs metadata neurodata:10.60178/cbs.20260429-001
 ```
 
 ### file-metadata
 
-Get the file metadata
+Get the file metadata. You can also specify a DOI path.
 
+```shell
+mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
+mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
+
+# DOI access examples:
+mdrs file-metadata "neurodata:10.60178/cbs.20260429-001/Figure 1/Figure1v3.pdf"
 ```
-$ mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
+
-$ mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
+### version
+
+Show the tool name and version number
+
+```shell
+mdrs version
 ```
 
 ### help
 
 Show the help message and exit
 
+```shell
+mdrs -h
 ```
-$ mdrs -h
+
+## Python API Usage
+
+You can also use this package as a Python library to programmatically interact with MDRS repositories.
+
+```python
+from mdrsclient.client import MdrsClient
+from mdrsclient.cache import InMemoryCache
+from mdrsclient.config import InMemoryConfig
+
+# 1. Setup in-memory configuration and cache to avoid local state files (e.g., config.ini, cache/*.json)
+config = InMemoryConfig("neurodata")
+config.url = "https://neurodata.riken.jp/api"
+
+cache = InMemoryCache()
+
+# 2. Initialize client with custom configuration and cache
+client = MdrsClient.from_remote("neurodata", cache=cache, config=config)
+
+# 3. Login to the remote server
+client.login("username", "password")
+
+# 4. Use service methods
+labs = client.get_laboratories()
+metadata = client.metadata("neurodata:/NIU/Repository/")
+
+# Transfer files programmatically
+client.upload("/path/to/local/data", "neurodata:/NIU/Repository/TEST/", is_recursive=True)
+client.download("neurodata:/NIU/Repository/TEST/data", "/path/to/local", is_recursive=True)
 ```
+
+## Testing
+
+You can run the unit test suite using the standard library `unittest` discover runner:
+
+```shell
+.venv/bin/python -m unittest discover tests
+```
+
+## Changelog
+
+See [CHANGELOG.md](./CHANGELOG.md) for the full change history.
+

mdrsclient/VERSION

@@ -1 +0,0 @@
-1.1.1

mdrsclient/__init__.py

@@ -1,3 +1,4 @@
 from mdrsclient.__version__ import __version__
+from mdrsclient.client import MdrsClient
 
-__all__ = ["__version__"]
+__all__ = ["__version__", "MdrsClient"]

mdrsclient/__main__.py

@@ -1,5 +1,6 @@
 import argparse
 import sys
+from json import JSONDecodeError
 
 from mdrsclient.commands import (
     ChaclCommand,
@@ -16,6 +17,7 @@ from mdrsclient.commands import (
     MvCommand,
     RmCommand,
     UploadCommand,
+    VersionCommand,
     WhoamiCommand,
 )
 from mdrsclient.exceptions import MDRSException
@@ -28,6 +30,7 @@ def main() -> None:
     parsers = parser.add_subparsers(title="subcommands")
 
     ConfigCommand.register(parsers)
+    VersionCommand.register(parsers)
     LoginCommand.register(parsers)
     LogoutCommand.register(parsers)
     WhoamiCommand.register(parsers)
@@ -52,6 +55,9 @@ def main() -> None:
     except MDRSException as e:
         print(f"Error: {e}")
         sys.exit(2)
+    except JSONDecodeError:
+        print("Unexpected response returned. Please check the configuration or the server's operational status.")
+        sys.exit(2)
     except KeyboardInterrupt:
         sys.exit(130)
 

mdrsclient/__version__.py

@@ -1,8 +1,9 @@
-import os
+import importlib.metadata
 
-here = os.path.realpath(os.path.dirname(__file__))
+try:
+    __version__ = importlib.metadata.version("mdrs-client-python")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.0.0-dev"
 
-with open(os.path.join(here, "VERSION")) as version_file:
-    __version__ = version_file.read().strip()
 
 __all__ = ["__version__"]

mdrsclient/api/__init__.py

@@ -1,11 +1,13 @@
-from mdrsclient.api.file import FileApi
+from mdrsclient.api.doi import DoiApi
-from mdrsclient.api.folder import FolderApi
+from mdrsclient.api.files import FilesApi
-from mdrsclient.api.laboratory import LaboratoryApi
+from mdrsclient.api.folders import FoldersApi
-from mdrsclient.api.user import UserApi
+from mdrsclient.api.laboratories import LaboratoriesApi
+from mdrsclient.api.users import UsersApi
 
 __all__ = [
-    "FileApi",
+    "DoiApi",
-    "FolderApi",
+    "FilesApi",
-    "LaboratoryApi",
+    "FoldersApi",
-    "UserApi",
+    "LaboratoriesApi",
+    "UsersApi",
 ]

mdrsclient/api/base.py

@@ -8,6 +8,7 @@ from mdrsclient.connection import MDRSConnection
 from mdrsclient.exceptions import (
     BadRequestException,
     ForbiddenException,
+    TokenExpiredException,
     UnauthorizedException,
     UnexpectedException,
 )
@@ -28,6 +29,11 @@ class BaseApi(ABC):
             if response.status_code == requests.codes.bad_request:
                 raise BadRequestException(errors.errors[0].detail)
             elif response.status_code == requests.codes.unauthorized:
+                # A request can wait in the server's queue for longer than the access
+                # token it was sent with lives, so an expired token here does not mean
+                # the session is over - it means this one request arrived too late.
+                if any(e.code == "token_not_valid" for e in errors.errors):
+                    raise TokenExpiredException("Access token expired before the request was served.")
                 raise UnauthorizedException("Login required.")
             elif response.status_code == requests.codes.forbidden:
                 raise ForbiddenException("You do not have enough permissions. Access is denied.")

mdrsclient/api/doi.py

@@ -0,0 +1,34 @@
+from typing import Final
+
+from pydantic import TypeAdapter
+from pydantic.dataclasses import dataclass
+
+from mdrsclient.api.base import BaseApi
+from mdrsclient.api.utils import retry_on_expired_token, token_check
+from mdrsclient.models.doi import Doi
+
+
+@dataclass(frozen=True)
+class DoiRetrieveFolderRef:
+    id: str
+
+
+@dataclass(frozen=True)
+class DoiRetrieveResponse:
+    id: str
+    doi: str
+    folder: DoiRetrieveFolderRef
+
+
+class DoiApi(BaseApi):
+    ENTRYPOINT: Final[str] = "v3/doi/"
+
+    @retry_on_expired_token
+    def retrieve(self, doi_id: str) -> Doi:
+        """Retrieve the folder associated with a DOI suffix ID (GET v3/doi/{id}/)."""
+        url = self.ENTRYPOINT + doi_id + "/"
+        token_check(self.connection)
+        response = self.connection.get(url)
+        self._raise_response_error(response)
+        api_resp = TypeAdapter(DoiRetrieveResponse).validate_python(response.json())
+        return Doi(id=api_resp.id, doi=api_resp.doi, folder_id=api_resp.folder.id)

mdrsclient/api/file.py

@@ -1,107 +0,0 @@
-from typing import Any, Final
-
-from pydantic import TypeAdapter
-from pydantic.dataclasses import dataclass
-
-from mdrsclient.api.base import BaseApi
-from mdrsclient.api.utils import token_check
-from mdrsclient.exceptions import UnexpectedException
-from mdrsclient.models import File
-
-
-@dataclass(frozen=True)
-class FileCreateResponse:
-    id: str
-
-
-class FileApi(BaseApi):
-    ENTRYPOINT: Final[str] = "v2/file/"
-
-    def retrieve(self, id: str) -> File:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + id + "/"
-        token_check(self.connection)
-        response = self.connection.get(url)
-        self._raise_response_error(response)
-        return TypeAdapter(File).validate_python(response.json())
-
-    def create(self, folder_id: str, path: str) -> str:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT
-        token_check(self.connection)
-        data: dict[str, str | int] = {"folder_id": folder_id}
-        try:
-            with open(path, mode="rb") as fp:
-                response = self.connection.post(url, data=data, files={"file": fp})
-                self._raise_response_error(response)
-                ret = TypeAdapter(FileCreateResponse).validate_python(response.json())
-        except OSError:
-            raise UnexpectedException(f"Could not open `{path}` file.")
-        return ret.id
-
-    def update(self, file: File, path: str | None) -> bool:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + file.id + "/"
-        token_check(self.connection)
-        if path is not None:
-            # update file body
-            try:
-                with open(path, mode="rb") as fp:
-                    response = self.connection.put(url, files={"file": fp})
-            except OSError:
-                raise UnexpectedException(f"Could not open `{path}` file.")
-        else:
-            # update metadata
-            data: dict[str, str | int] = {"name": file.name, "description": file.description}
-            response = self.connection.put(url, data=data)
-        self._raise_response_error(response)
-        return True
-
-    def destroy(self, file: File) -> bool:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + file.id + "/"
-        token_check(self.connection)
-        response = self.connection.delete(url)
-        self._raise_response_error(response)
-        return True
-
-    def move(self, file: File, folder_id: str, name: str) -> bool:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + file.id + "/move/"
-        data: dict[str, str | int] = {"folder": folder_id, "name": name}
-        token_check(self.connection)
-        response = self.connection.post(url, data=data)
-        self._raise_response_error(response)
-        return True
-
-    def copy(self, file: File, folder_id: str, name: str) -> bool:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + file.id + "/copy/"
-        data: dict[str, str | int] = {"folder": folder_id, "name": name}
-        token_check(self.connection)
-        response = self.connection.post(url, data=data)
-        self._raise_response_error(response)
-        return True
-
-    def metadata(self, file: File) -> dict[str, Any]:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + file.id + "/metadata/"
-        token_check(self.connection)
-        response = self.connection.get(url)
-        self._raise_response_error(response)
-        return response.json()
-
-    def download(self, file: File, path: str) -> bool:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = "v2/" + file.download_url
-        response = self.connection.get(url, stream=True)
-        self._raise_response_error(response)
-        try:
-            with open(path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=4096):
-                    if chunk:
-                        f.write(chunk)
-                        f.flush()
-        except PermissionError:
-            print(f"Cannot create file `{path}`: Permission denied.")
-        return True

mdrsclient/api/files.py

@@ -0,0 +1,201 @@
+import mimetypes
+import os
+import threading
+from typing import Any, Final
+from unicodedata import normalize
+
+from pydantic import TypeAdapter
+from pydantic.dataclasses import dataclass
+from requests_toolbelt.multipart.encoder import MultipartEncoder
+
+from mdrsclient.api.base import BaseApi
+from mdrsclient.api.utils import retry_on_expired_token, token_check
+from mdrsclient.exceptions import MDRSException, UnexpectedException
+from mdrsclient.models import File
+
+
+@dataclass(frozen=True)
+class FilesApiCreateResponse:
+    id: str
+
+
+@dataclass(frozen=True)
+class FilesApiListResponse:
+    count: int
+    next: str | None
+    previous: str | None
+    results: list[File]
+
+
+class FilesApi(BaseApi):
+    ENTRYPOINT: Final[str] = "v3/files/"
+    FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
+
+    @retry_on_expired_token
+    def list(self, folder_id: str, page_num: int) -> FilesApiListResponse:
+        url = self.ENTRYPOINT
+        token_check(self.connection)
+        params: dict[str, str | int] = {"folder_id": folder_id, "page": page_num}
+        response = self.connection.get(url, params=params)
+        self._raise_response_error(response)
+        return TypeAdapter(FilesApiListResponse).validate_python(response.json())
+
+    @retry_on_expired_token
+    def retrieve(self, id: str) -> File:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + id + "/"
+        token_check(self.connection)
+        response = self.connection.get(url)
+        self._raise_response_error(response)
+        return TypeAdapter(File).validate_python(response.json())
+
+    @retry_on_expired_token
+    def create(self, folder_id: str, path: str) -> str:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT
+        token_check(self.connection)
+        data: dict[str, str | int] | MultipartEncoder = {}
+        try:
+            with open(os.path.realpath(path), mode="rb") as fp:
+                data = MultipartEncoder(
+                    fields={
+                        "folder_id": folder_id,
+                        "file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path)),
+                    }
+                )
+                response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
+                self._raise_response_error(response)
+                ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
+        except OSError:
+            raise UnexpectedException(f"Could not open `{path}` file.")
+        except MemoryError:
+            raise UnexpectedException("Out of memory.")
+        except MDRSException:
+            # Already says what went wrong, and the caller may want to act on the kind.
+            raise
+        except Exception as e:
+            raise UnexpectedException("Unspecified error.") from e
+        return ret.id
+
+    @retry_on_expired_token
+    def update(self, file: File, path: str | None) -> bool:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + file.id + "/"
+        token_check(self.connection)
+        data: dict[str, str | int] | MultipartEncoder = {}
+        if path is not None:
+            # update file body
+            try:
+                with open(os.path.realpath(path), mode="rb") as fp:
+                    data = MultipartEncoder(
+                        fields={"file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path))}
+                    )
+                    response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
+            except OSError:
+                raise UnexpectedException(f"Could not open `{path}` file.")
+            except MemoryError:
+                raise UnexpectedException("Out of memory.")
+            except Exception as e:
+                raise UnexpectedException("Unspecified error.") from e
+        else:
+            # update metadata
+            data = {"name": file.name, "description": file.description}
+            response = self.connection.put(url, data=data)
+        self._raise_response_error(response)
+        return True
+
+    @retry_on_expired_token
+    def destroy(self, file: File) -> bool:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + file.id + "/"
+        token_check(self.connection)
+        response = self.connection.delete(url)
+        self._raise_response_error(response)
+        return True
+
+    @retry_on_expired_token
+    def move(self, file: File, folder_id: str, name: str) -> bool:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + file.id + "/move/"
+        data: dict[str, str | int] = {"folder": folder_id, "name": name}
+        token_check(self.connection)
+        response = self.connection.post(url, data=data)
+        self._raise_response_error(response)
+        return True
+
+    @retry_on_expired_token
+    def copy(self, file: File, folder_id: str, name: str) -> bool:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + file.id + "/copy/"
+        data: dict[str, str | int] = {"folder": folder_id, "name": name}
+        token_check(self.connection)
+        response = self.connection.post(url, data=data)
+        self._raise_response_error(response)
+        return True
+
+    @retry_on_expired_token
+    def metadata(self, file: File) -> dict[str, Any]:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + file.id + "/metadata/"
+        token_check(self.connection)
+        response = self.connection.get(url)
+        self._raise_response_error(response)
+        return response.json()
+
+    @retry_on_expired_token
+    def download(self, file: File, path: str) -> bool:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = file.download_url
+        token_check(self.connection)
+        # Refused before anything is fetched. The finished file is moved into place, and a
+        # rename would replace a destination whose mode says it is protected.
+        if os.path.exists(path):
+            try:
+                with open(path, "r+b"):
+                    pass
+            except OSError as e:
+                raise UnexpectedException(f"Cannot write `{path}`: {e}")
+        response = self.connection.get(url, stream=True)
+        self._raise_response_error(response)
+        # Written beside the destination and moved in once the whole body has arrived, so
+        # a transfer that fails part way leaves whatever was already there untouched and
+        # never leaves a truncated file under the real name.
+        fd, tmp_path = self._open_partial(path)
+        try:
+            with os.fdopen(fd, "wb") as f:
+                for chunk in response.iter_content(chunk_size=4096):
+                    if chunk:
+                        f.write(chunk)
+            os.replace(tmp_path, path)
+        except BaseException:
+            # Only the scratch file goes: anything at the destination was not written here.
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+            raise
+        return True
+
+    @staticmethod
+    def _open_partial(path: str) -> tuple[int, str]:
+        """
+        Create a scratch file beside `path` and return it open for writing.
+
+        Beside it, so moving the finished download into place is a rename within one
+        directory. `0o666` rather than a private mode because the umask is what decided
+        the permissions of a downloaded file before, and still should.
+        """
+        base = f"{path}.{os.getpid()}-{threading.get_ident()}"
+        for attempt in range(100):
+            tmp_path = f"{base}-{attempt}.mdrspart"
+            try:
+                return os.open(tmp_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666), tmp_path
+            except FileExistsError:
+                continue
+            except OSError as e:
+                raise UnexpectedException(f"Cannot write `{path}`: {e}")
+        raise UnexpectedException(f"Could not create a temporary file beside `{path}`.")
+
+    def _get_mime_type(self, path: str) -> str:
+        mt = mimetypes.guess_type(path)
+        if mt:
+            return mt[0] or self.FALLBACK_MIMETYPE
+        return self.FALLBACK_MIMETYPE

mdrsclient/api/folders.py

@@ -5,19 +5,20 @@ from pydantic import TypeAdapter
 from pydantic.dataclasses import dataclass
 
 from mdrsclient.api.base import BaseApi
-from mdrsclient.api.utils import token_check
+from mdrsclient.api.utils import retry_on_expired_token, token_check
 from mdrsclient.exceptions import UnauthorizedException
 from mdrsclient.models import Folder, FolderSimple
 
 
 @dataclass(frozen=True)
-class FolderCreateResponse:
+class FoldersApiCreateResponse:
     id: str
 
 
-class FolderApi(BaseApi):
+class FoldersApi(BaseApi):
-    ENTRYPOINT: Final[str] = "v2/folder/"
+    ENTRYPOINT: Final[str] = "v3/folders/"
 
+    @retry_on_expired_token
     def list(self, laboratory_id: int, path: str) -> list[FolderSimple]:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT
@@ -30,6 +31,7 @@ class FolderApi(BaseApi):
             ret.append(TypeAdapter(FolderSimple).validate_python(data))
         return ret
 
+    @retry_on_expired_token
     def retrieve(self, id: str) -> Folder:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/"
@@ -39,6 +41,7 @@ class FolderApi(BaseApi):
         ret = TypeAdapter(Folder).validate_python(response.json())
         return ret
 
+    @retry_on_expired_token
     def create(self, name: str, parent_id: str) -> str:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT
@@ -46,9 +49,10 @@ class FolderApi(BaseApi):
         token_check(self.connection)
         response = self.connection.post(url, data=data)
         self._raise_response_error(response)
-        ret = TypeAdapter(FolderCreateResponse).validate_python(response.json())
+        ret = TypeAdapter(FoldersApiCreateResponse).validate_python(response.json())
         return ret.id
 
+    @retry_on_expired_token
     def update(self, folder: FolderSimple) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + folder.id + "/"
@@ -61,6 +65,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def destroy(self, id: str, recursive: bool) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/"
@@ -70,6 +75,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def auth(self, id: str, password: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/auth/"
@@ -81,6 +87,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def acl(self, id: str, access_level: int, recursive: bool, password: str | None) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/acl/"
@@ -94,6 +101,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def move(self, folder: FolderSimple, folder_id: str, name: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + folder.id + "/move/"
@@ -103,6 +111,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def copy(self, folder: FolderSimple, folder_id: str, name: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + folder.id + "/copy/"
@@ -112,6 +121,7 @@ class FolderApi(BaseApi):
         self._raise_response_error(response)
         return True
 
+    @retry_on_expired_token
     def metadata(self, id: str) -> dict[str, Any]:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/metadata/"

mdrsclient/api/laboratories.py

@@ -3,13 +3,14 @@ from typing import Final
 from pydantic import TypeAdapter
 
 from mdrsclient.api.base import BaseApi
-from mdrsclient.api.utils import token_check
+from mdrsclient.api.utils import retry_on_expired_token, token_check
 from mdrsclient.models import Laboratories, Laboratory
 
 
-class LaboratoryApi(BaseApi):
+class LaboratoriesApi(BaseApi):
-    ENTRYPOINT: Final[str] = "v2/laboratory/"
+    ENTRYPOINT: Final[str] = "v3/laboratories/"
 
+    @retry_on_expired_token
     def list(self) -> Laboratories:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT

mdrsclient/api/user.py

@@ -1,46 +0,0 @@
-from typing import Final
-
-import requests
-from pydantic import TypeAdapter
-from pydantic.dataclasses import dataclass
-
-from mdrsclient.api.base import BaseApi
-from mdrsclient.exceptions import UnauthorizedException
-from mdrsclient.models import Token, User
-
-
-@dataclass(frozen=True)
-class UserAuthResponse(Token):
-    is_reviewer: bool | None = None
-    laboratory: str | None = None
-    lab_id: int | None = None
-
-
-class UserApi(BaseApi):
-    ENTRYPOINT: Final[str] = "v2/"
-
-    def auth(self, username: str, password: str) -> tuple[User, Token]:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + "auth/"
-        data: dict[str, str | int] = {"username": username, "password": password}
-        response = self.connection.post(url, data=data)
-        if response.status_code == requests.codes.unauthorized:
-            raise UnauthorizedException("Invalid username or password.")
-        self._raise_response_error(response)
-        obj = TypeAdapter(UserAuthResponse).validate_python(response.json())
-        token = Token(access=obj.access, refresh=obj.refresh)
-        laboratory_ids = [obj.lab_id] if obj.lab_id is not None else []
-        is_reviewer = obj.is_reviewer if obj.is_reviewer is not None else False
-        user = User(id=token.user_id, username=username, laboratory_ids=laboratory_ids, is_reviewer=is_reviewer)
-        return (user, token)
-
-    def refresh(self, token: Token) -> Token:
-        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
-        url = self.ENTRYPOINT + "refresh/"
-        data: dict[str, str | int] = {"refresh": token.refresh}
-        response = self.connection.post(url, data=data)
-        if response.status_code == requests.codes.unauthorized:
-            raise UnauthorizedException("Token is invalid or expired.")
-        self._raise_response_error(response)
-        token = TypeAdapter(Token).validate_python(response.json())
-        return token

mdrsclient/api/users.py

@@ -0,0 +1,76 @@
+from typing import Final
+
+import requests
+from pydantic import TypeAdapter
+from pydantic.dataclasses import dataclass
+
+from mdrsclient.api.base import BaseApi
+from mdrsclient.exceptions import UnauthorizedException
+from mdrsclient.models import Token, User
+
+# (connect, read) seconds for the token refresh. Uploads are served by a separate
+# instance, so this one is not queued behind them and has no reason to be slow.
+TOKEN_REFRESH_TIMEOUT: Final[tuple[float, float]] = (5.0, 30.0)
+
+
+@dataclass(frozen=True)
+class UsersCurrentResponseLaboratory:
+    id: int
+    name: str
+    role: int
+
+
+@dataclass(frozen=True)
+class UsersApiCurrentResponse:
+    id: int
+    username: str
+    first_name: str
+    last_name: str
+    email: str
+    orcid_id: str
+    laboratories: list[UsersCurrentResponseLaboratory]
+    is_staff: bool
+    is_active: bool
+    is_superuser: bool
+    is_reviewer: bool
+    last_login: str  # ISO8601
+    date_joined: str  # ISO8601
+
+
+class UsersApi(BaseApi):
+    ENTRYPOINT: Final[str] = "v3/users/"
+
+    def current(self) -> User:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + "current/"
+        response = self.connection.get(url)
+        self._raise_response_error(response)
+        obj = TypeAdapter(UsersApiCurrentResponse).validate_python(response.json())
+        laboratory_ids = list(map(lambda x: x.id, obj.laboratories))
+        user = User(id=obj.id, username=obj.username, laboratory_ids=laboratory_ids, is_reviewer=obj.is_reviewer)
+        return user
+
+    def token(self, username: str, password: str) -> Token:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + "token/"
+        data: dict[str, str | int] = {"username": username, "password": password}
+        response = self.connection.post(url, data=data)
+        if response.status_code == requests.codes.unauthorized:
+            raise UnauthorizedException("Invalid username or password.")
+        self._raise_response_error(response)
+        token = TypeAdapter(Token).validate_python(response.json())
+        return token
+
+    def tokenRefresh(self, token: Token) -> Token:
+        # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
+        url = self.ENTRYPOINT + "token/refresh/"
+        data: dict[str, str | int] = {"refresh": token.refresh}
+        # Bounded on its own: the caller holds a lock that spans processes while this runs,
+        # so a provider that accepts the connection and then goes quiet would otherwise
+        # stall every other request on this machine rather than just this one.
+        response = self.connection.post(url, data=data, timeout=TOKEN_REFRESH_TIMEOUT)
+        if response.status_code == requests.codes.unauthorized:
+            raise UnauthorizedException("Token is invalid or expired.")
+        self._raise_response_error(response)
+        token = TypeAdapter(Token).validate_python(response.json())
+        return token

mdrsclient/api/utils.py

@@ -1,19 +1,124 @@
-from mdrsclient.api.user import UserApi
+import functools
+from typing import Any, Callable, TypeVar, cast
+
+from mdrsclient.api.users import UsersApi
 from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import UnauthorizedException
+from mdrsclient.exceptions import (
+    MDRSException,
+    ServerBusyException,
+    TokenExpiredException,
+    UnauthorizedException,
+)
+from mdrsclient.models import Token
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+SERVER_BUSY_MESSAGE = (
+    "The server took too long to start handling the request and may be overloaded. "
+    "Try again, or reduce the number of parallel transfers."
+)
 
 
 def token_check(connection: MDRSConnection) -> None:
+    """
+    Bring the access token up to date before a request goes out.
+
+    Refreshing is a read-modify-write over a cache shared with every other client
+    process on this machine, and a rotating provider stops honouring the refresh token
+    it replaces. Two processes reaching this at once would otherwise both send the same
+    token, and the loser would be left holding one the server no longer accepts, so the
+    whole sequence runs under a lock that spans processes and the cache is re-read
+    inside it.
+    """
+    with connection.lock:
+        token = connection.token
+        if token is None or not (token.is_refresh_required or token.is_expired):
+            # Nothing to do, which is the answer for almost every request. The lock below
+            # reaches across processes and is held for a round trip, so it is worth
+            # knowing that before taking it.
+            return
+        with connection.cache_lock():
+            connection.reload_cache()
+            token = connection.token
+            if token is None:
+                return
+            if token.is_expired:
+                connection.logout()
+                return
+            if not token.is_refresh_required:
+                return
+            user_api = UsersApi(connection)
             try:
-        connection.lock.acquire()
+                connection.token = user_api.tokenRefresh(token)
-        if connection.token is not None:
-            if connection.token.is_refresh_required:
-                user_api = UserApi(connection)
-                try:
-                    connection.token = user_api.refresh(connection.token)
             except UnauthorizedException:
                 connection.logout()
-            elif connection.token.is_expired:
+
+
+def token_recover(connection: MDRSConnection, used: Token | None) -> bool:
+    """
+    Get a usable access token after one was refused for having expired.
+
+    A request can sit in the server's queue for longer than its access token lives, and
+    while it waited another thread or process may already have refreshed. Prefer what
+    they left behind: refreshing again would spend a round trip, and with a provider
+    that rotates refresh tokens it would retire one that is still in use.
+
+    Returns whether the caller now holds a token worth retrying with.
+    """
+    with connection.lock, connection.cache_lock():
+        connection.reload_cache()
+        token = connection.token
+        if token is None:
+            return False
+        if used is None or token != used:
+            # Somebody else has been here since the request went out. Assign rather than
+            # just read: the setter is what rewrites the session's Authorization header,
+            # so a plain reload would leave the retry carrying the refused token.
+            connection.token = token
+            return True
+        user_api = UsersApi(connection)
+        try:
+            connection.token = user_api.tokenRefresh(token)
+        except UnauthorizedException:
             connection.logout()
-    finally:
+            return False
-        connection.lock.release()
+        return True
+
+
+def retry_on_expired_token(func: F) -> F:
+    """
+    Send a request again once when the server found its access token expired.
+
+    The token is checked before every request, so this only happens when the request was
+    held long enough on the way in for a valid token to lapse - a queue behind uploads
+    that take minutes, most often.
+    """
+
+    @functools.wraps(func)
+    def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+        connection: MDRSConnection = self.connection
+        # Settle the token first so `used` is what the request actually carries. The
+        # wrapped method checks it again, which costs nothing once it is already current.
+        token_check(connection)
+        used = connection.token
+        try:
+            return func(self, *args, **kwargs)
+        except TokenExpiredException:
+            try:
+                recovered = token_recover(connection, used)
+            except MDRSException:
+                raise
+            except Exception as e:
+                # A busy server queues the refresh too, and failing here would report an
+                # endpoint the user never asked for rather than the reason.
+                raise ServerBusyException(SERVER_BUSY_MESSAGE) from e
+            if not recovered:
+                raise
+        try:
+            return func(self, *args, **kwargs)
+        except TokenExpiredException as e:
+            # A second refusal, with a token that was current when it left. The session is
+            # fine; the server is not starting requests before their credentials lapse.
+            raise ServerBusyException(SERVER_BUSY_MESSAGE) from e
+
+    return cast(F, wrapper)

mdrsclient/cache.py

@@ -1,7 +1,12 @@
+import contextlib
 import dataclasses
 import hashlib
 import json
 import os
+import tempfile
+import time
+from contextlib import AbstractContextManager
+from typing import Iterator, Protocol, runtime_checkable
 
 from pydantic import TypeAdapter, ValidationError
 from pydantic.dataclasses import dataclass
@@ -16,7 +21,7 @@ from mdrsclient.utils import FileLock
 class CacheData:
     user: User | None = None
     token: Token | None = None
-    laboratories: Laboratories = Laboratories()
+    laboratories: Laboratories = dataclasses.field(default_factory=Laboratories)
     digest: str = ""
 
     def clear(self) -> None:
@@ -43,18 +48,132 @@ class CacheData:
         ).hexdigest()
 
 
-class CacheFile:
+@runtime_checkable
+class CacheInterface(Protocol):
+    def lock(self) -> AbstractContextManager[None]:
+        """
+        Hold exclusive access to the cache for the duration of the block.
+
+        Refreshing a token is a read-modify-write, and the cache is shared with every
+        other client process using the same remote.
+        """
+        ...
+
+    def reload(self) -> None:
+        """Re-read the cache, discarding anything held from an earlier read."""
+        ...
+
+    @property
+    def token(self) -> Token | None: ...
+    @token.setter
+    def token(self, token: Token) -> None: ...
+    @token.deleter
+    def token(self) -> None: ...
+
+    @property
+    def user(self) -> User | None: ...
+    @user.setter
+    def user(self, user: User) -> None: ...
+    @user.deleter
+    def user(self) -> None: ...
+
+    @property
+    def laboratories(self) -> Laboratories: ...
+    @laboratories.setter
+    def laboratories(self, laboratories: Laboratories) -> None: ...
+
+
+class InMemoryCache(CacheInterface):
+    def __init__(self) -> None:
+        self.__data = CacheData()
+
+    @contextlib.contextmanager
+    def lock(self) -> Iterator[None]:
+        # Nothing else can reach this cache, so there is nothing to exclude.
+        yield
+
+    def reload(self) -> None:
+        pass
+
+    @property
+    def token(self) -> Token | None:
+        return self.__data.token
+
+    @token.setter
+    def token(self, token: Token) -> None:
+        self.__data.token = token
+
+    @token.deleter
+    def token(self) -> None:
+        if self.__data.token is not None:
+            self.__data.token = None
+
+    @property
+    def user(self) -> User | None:
+        return self.__data.user
+
+    @user.setter
+    def user(self, user: User) -> None:
+        self.__data.user = user
+
+    @user.deleter
+    def user(self) -> None:
+        if self.__data.user is not None:
+            self.__data.user = None
+
+    @property
+    def laboratories(self) -> Laboratories:
+        return self.__data.laboratories
+
+    @laboratories.setter
+    def laboratories(self, laboratories: Laboratories) -> None:
+        self.__data.laboratories = laboratories
+
+
+class CacheFile(CacheInterface):
     __serial: int
     __cache_dir: str
     __cache_file: str
+    __lock_file: str
+    __lock_depth: int
     __data: CacheData
 
     def __init__(self, remote: str) -> None:
         self.__serial = -1
         self.__cache_dir = os.path.join(CONFIG_DIRNAME, "cache")
         self.__cache_file = os.path.join(self.__cache_dir, remote + ".json")
+        self.__lock_file = os.path.join(self.__cache_dir, remote + ".lock")
+        self.__lock_depth = 0
         self.__data = CacheData()
 
+    @contextlib.contextmanager
+    def lock(self) -> Iterator[None]:
+        # Re-entrant, because every write takes it and a refresh is a write made while
+        # already holding it. A second flock on the same file from the same process would
+        # wait for a release that cannot come.
+        if self.__lock_depth > 0:
+            self.__lock_depth += 1
+            try:
+                yield
+            finally:
+                self.__lock_depth -= 1
+            return
+        # A separate file, so that replacing the cache cannot disturb the lock holders.
+        self.__ensure_cache_dir()
+        with open(self.__lock_file, "a") as f:
+            FileLock.lock(f)
+            self.__lock_depth = 1
+            try:
+                yield
+            finally:
+                self.__lock_depth = 0
+                FileLock.unlock(f)
+        os.chmod(self.__lock_file, 0o600)
+
+    def reload(self) -> None:
+        self.__serial = -1
+        self.__load()
+
     @property
     def token(self) -> Token | None:
         self.__load()
@@ -62,7 +181,8 @@ class CacheFile:
 
     @token.setter
     def token(self, token: Token) -> None:
-        self.__load()
+        with self.lock():
+            self.reload()
             self.__data.token = token
             self.__save()
 
@@ -77,7 +197,8 @@ class CacheFile:
 
     @user.setter
     def user(self, user: User) -> None:
-        self.__load()
+        with self.lock():
+            self.reload()
             self.__data.user = user
             self.__save()
 
@@ -92,7 +213,8 @@ class CacheFile:
 
     @laboratories.setter
     def laboratories(self, laboratories: Laboratories) -> None:
-        self.__load()
+        with self.lock():
+            self.reload()
             self.__data.laboratories = laboratories
             self.__save()
 
@@ -102,8 +224,7 @@ class CacheFile:
 
     def __load(self) -> None:
         if os.path.isfile(self.__cache_file):
-            stat = os.stat(self.__cache_file)
+            serial = self.__stat_serial()
-            serial = hash((stat.st_uid, stat.st_gid, stat.st_mode, stat.st_size, stat.st_mtime))
             if self.__serial != serial:
                 try:
                     with open(self.__cache_file) as f:
@@ -123,15 +244,44 @@ class CacheFile:
 
     def __save(self) -> None:
         self.__ensure_cache_dir()
-        with open(self.__cache_file, "w") as f:
-            FileLock.lock(f)
         self.__data.update_digest()
-            f.write(json.dumps(dataclasses.asdict(self.__data)))
+        payload = json.dumps(dataclasses.asdict(self.__data))
-            FileLock.unlock(f)
+        # Written aside and moved into place: opening the cache for writing truncates it
-        stat = os.stat(self.__cache_file)
+        # first, and a reader landing in that window would find the file empty and treat
-        self.__serial = hash((stat.st_uid, stat.st_gid, stat.st_mode, stat.st_size, stat.st_mtime))
+        # the session as broken.
+        fd, tmp_file = tempfile.mkstemp(dir=self.__cache_dir, prefix=".tmp-")
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(payload)
             # ensure file is secure.
-        os.chmod(self.__cache_file, 0o600)
+            os.chmod(tmp_file, 0o600)
+            self.__replace(tmp_file, self.__cache_file)
+        except BaseException:
+            if os.path.exists(tmp_file):
+                os.unlink(tmp_file)
+            raise
+        self.__serial = self.__stat_serial()
+
+    @staticmethod
+    def __replace(source: str, destination: str) -> None:
+        # Windows refuses the rename while another process still has the destination
+        # open, which a reader briefly does, so give it a moment rather than failing the
+        # save outright. On POSIX the rename always succeeds and the loop ends at once.
+        deadline = time.monotonic() + 5.0
+        while True:
+            try:
+                os.replace(source, destination)
+                return
+            except PermissionError:
+                if time.monotonic() >= deadline:
+                    raise
+                time.sleep(0.05)
+
+    def __stat_serial(self) -> int:
+        stat = os.stat(self.__cache_file)
+        # st_ino and st_mtime_ns both move when the file is replaced, which a refresh
+        # that happens to produce the same number of bytes otherwise would not show.
+        return hash((stat.st_ino, stat.st_uid, stat.st_gid, stat.st_mode, stat.st_size, stat.st_mtime_ns))
 
     def __ensure_cache_dir(self) -> None:
         if not os.path.exists(self.__cache_dir):

mdrsclient/client.py

@@ -0,0 +1,253 @@
+import os
+from dataclasses import dataclass
+from unicodedata import normalize
+
+from mdrsclient.api import FilesApi, FoldersApi
+from mdrsclient.cache import CacheInterface
+from mdrsclient.config import ConfigInterface
+from mdrsclient.connection import MDRSConnection
+from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models import File, Folder, FolderSimple, Laboratory
+from mdrsclient.models.file import find_file
+from mdrsclient.services import MdrsService
+
+
+@dataclass(frozen=True)
+class TransferEndpoints:
+    """Both ends of a `cp` or `mv`, resolved: the parents, their contents, and the names."""
+
+    laboratory: Laboratory
+    s_path: str
+    s_basename: str
+    s_parent_folder: Folder
+    s_parent_files: list[File]
+    d_path: str
+    d_basename: str
+    d_parent_folder: Folder
+    d_parent_files: list[File]
+
+    def is_same_place(self) -> bool:
+        """Whether the destination names what the source already is, leaving nothing to do."""
+        return self.s_parent_folder.id == self.d_parent_folder.id and self.s_basename == self.d_basename
+
+
+class MdrsClient(MdrsService):
+    """Service layer client for MDRS."""
+
+    def __init__(self, connection: MDRSConnection | None, config_class: type[ConfigInterface] | None = None):
+        super().__init__(connection, config_class)
+
+    @classmethod
+    def from_remote(
+        cls, remote: str, cache: CacheInterface | None = None, config: ConfigInterface | None = None
+    ) -> "MdrsClient":
+        return cls(cls.create_connection(remote, cache, config))
+
+    def mkdir(self, remote_path: str) -> None:
+        remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
+        r_path = r_path.rstrip("/")
+        r_dirname = os.path.dirname(r_path)
+        r_basename = os.path.basename(r_path)
+        laboratory = self.find_laboratory(laboratory_name)
+        parent_folder = self.find_folder(laboratory, r_dirname)
+        files = self.find_files(parent_folder.id)
+        if parent_folder.find_sub_folder(r_basename) is not None or find_file(files, r_basename) is not None:
+            raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
+        folder_api = FoldersApi(self.connection)
+        folder_api.create(normalize("NFC", r_basename), parent_folder.id)
+
+    def rm(self, remote_path: str, is_recursive: bool = False) -> None:
+        remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
+        r_path = r_path.rstrip("/")
+        r_dirname = os.path.dirname(r_path)
+        r_basename = os.path.basename(r_path)
+        laboratory = self.find_laboratory(laboratory_name)
+        parent_folder = self.find_folder(laboratory, r_dirname)
+        parent_files = self.find_files(parent_folder.id)
+        file = find_file(parent_files, r_basename)
+        if file is not None:
+            file_api = FilesApi(self.connection)
+            file_api.destroy(file)
+        else:
+            folder = parent_folder.find_sub_folder(r_basename)
+            if folder is None:
+                raise IllegalArgumentException(f"Cannot remove `{r_path}`: No such file or folder.")
+            if not is_recursive:
+                raise IllegalArgumentException(f"Cannot remove `{r_path}`: Is a folder.")
+            folder_api = FoldersApi(self.connection)
+            folder_api.destroy(folder.id, True)
+
+    def ls(self, remote_path: str, password: str | None = None) -> tuple[Folder, list[File]]:
+        folder, laboratory = self.resolve_folder(remote_path, password)
+        files = self.find_files(folder.id)
+        return folder, files
+
+    def __resolve_transfer(self, src_path: str, dest_path: str) -> TransferEndpoints:
+        """Resolve both ends of a transfer, refusing one that crosses a remote or a laboratory."""
+        s_remote, s_laboratory_name, s_path = self.parse_remote_host_with_path(src_path)
+        d_remote, d_laboratory_name, d_path = self.parse_remote_host_with_path(dest_path)
+        if s_remote != d_remote:
+            raise IllegalArgumentException("Remote host mismatched.")
+        if s_laboratory_name != d_laboratory_name:
+            raise IllegalArgumentException("Laboratory mismatched.")
+        s_path = s_path.rstrip("/")
+        s_basename = os.path.basename(s_path)
+        # A destination ending in a separator names a folder to put the source into, and
+        # keeps the source's own name.
+        if d_path.endswith("/"):
+            d_dirname = d_path
+            d_basename = s_basename
+        else:
+            d_dirname = os.path.dirname(d_path)
+            d_basename = os.path.basename(d_path)
+        laboratory = self.find_laboratory(s_laboratory_name)
+        s_parent_folder = self.find_folder(laboratory, os.path.dirname(s_path))
+        d_parent_folder = self.find_folder(laboratory, d_dirname)
+        return TransferEndpoints(
+            laboratory=laboratory,
+            s_path=s_path,
+            s_basename=s_basename,
+            s_parent_folder=s_parent_folder,
+            s_parent_files=self.find_files(s_parent_folder.id),
+            d_path=d_path,
+            d_basename=d_basename,
+            d_parent_folder=d_parent_folder,
+            d_parent_files=self.find_files(d_parent_folder.id),
+        )
+
+    @staticmethod
+    def __check_file_destination(ends: TransferEndpoints) -> None:
+        """Refuse a destination that a file cannot take the place of."""
+        if find_file(ends.d_parent_files, ends.d_basename) is not None:
+            raise IllegalArgumentException(f"File `{ends.d_basename}` already exists.")
+        if ends.d_parent_folder.find_sub_folder(ends.d_basename) is not None:
+            raise IllegalArgumentException(
+                f"Cannot overwrite non-folder `{ends.d_basename}` with folder `{ends.d_path}`."
+            )
+
+    @staticmethod
+    def __check_folder_destination(ends: TransferEndpoints, s_folder: FolderSimple) -> None:
+        """Refuse a destination that a folder cannot take the place of."""
+        if find_file(ends.d_parent_files, ends.d_basename) is not None:
+            raise IllegalArgumentException(
+                f"Cannot overwrite non-folder `{ends.d_basename}` with folder `{ends.s_path}`."
+            )
+        d_folder = ends.d_parent_folder.find_sub_folder(ends.d_basename)
+        if d_folder is not None:
+            if d_folder.id == s_folder.id:
+                raise IllegalArgumentException(f"`{ends.s_path}` and `{ends.s_path}` are the same folder.")
+            raise IllegalArgumentException(f"Cannot move `{ends.s_path}` to `{ends.d_path}`: Folder not empty.")
+
+    @staticmethod
+    def __find_source_folder(ends: TransferEndpoints) -> FolderSimple:
+        s_folder = ends.s_parent_folder.find_sub_folder(ends.s_basename)
+        if s_folder is None:
+            raise IllegalArgumentException(f"File or folder `{ends.s_basename}` not found.")
+        return s_folder
+
+    def cp(self, src_path: str, dest_path: str, is_recursive: bool = False) -> None:
+        ends = self.__resolve_transfer(src_path, dest_path)
+        s_file = find_file(ends.s_parent_files, ends.s_basename)
+        if s_file is not None:
+            self.__check_file_destination(ends)
+            if not ends.is_same_place():
+                FilesApi(self.connection).copy(s_file, ends.d_parent_folder.id, normalize("NFC", ends.d_basename))
+            return
+        s_folder = self.__find_source_folder(ends)
+        if not is_recursive:
+            raise IllegalArgumentException(f"Cannot copy `{ends.s_path}`: Is a folder.")
+        self.__check_folder_destination(ends, s_folder)
+        if not ends.is_same_place():
+            FoldersApi(self.connection).copy(s_folder, ends.d_parent_folder.id, normalize("NFC", ends.d_basename))
+
+    def mv(self, src_path: str, dest_path: str) -> None:
+        ends = self.__resolve_transfer(src_path, dest_path)
+        s_file = find_file(ends.s_parent_files, ends.s_basename)
+        if s_file is not None:
+            self.__check_file_destination(ends)
+            if not ends.is_same_place():
+                FilesApi(self.connection).move(s_file, ends.d_parent_folder.id, normalize("NFC", ends.d_basename))
+            return
+        s_folder = self.__find_source_folder(ends)
+        self.__check_folder_destination(ends, s_folder)
+        if not ends.is_same_place():
+            FoldersApi(self.connection).move(s_folder, ends.d_parent_folder.id, normalize("NFC", ends.d_basename))
+
+    def chacl(
+        self, remote_path: str, access_level: int, is_recursive: bool = False, password: str | None = None
+    ) -> None:
+        remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
+        r_path = r_path.rstrip("/")
+        laboratory = self.find_laboratory(laboratory_name)
+        folder = self.find_folder(laboratory, r_path)
+        folder_api = FoldersApi(self.connection)
+        folder_api.acl(folder.id, access_level, is_recursive, password)
+
+    def metadata(self, remote_path: str, password: str | None = None) -> dict:
+        folder, laboratory = self.resolve_folder(remote_path, password)
+        folder_api = FoldersApi(self.connection)
+        return folder_api.metadata(folder.id)
+
+    def file_metadata(self, remote_path: str, password: str | None = None) -> dict:
+        folder, laboratory, r_basename = self.resolve_file(remote_path, password)
+        files = self.find_files(folder.id)
+        file = find_file(files, r_basename)
+        if file is None:
+            raise IllegalArgumentException(f"File `{r_basename}` not found.")
+        file_api = FilesApi(self.connection)
+        return file_api.metadata(file)
+
+    def upload(
+        self, local_path: str, remote_path: str, is_recursive: bool = False, is_skip_if_exists: bool = False
+    ) -> None:
+        from mdrsclient.transfer import Uploader
+
+        uploader = Uploader(self)
+        uploader.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
+
+    def download(
+        self,
+        remote_path: str,
+        local_path: str,
+        is_recursive: bool = False,
+        is_skip_if_exists: bool = False,
+        password: str | None = None,
+        excludes: list[str] | None = None,
+    ) -> None:
+        from mdrsclient.transfer import Downloader
+
+        downloader = Downloader(self)
+        downloader.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
+
+    def version(self) -> str:
+        from mdrsclient.__version__ import __version__
+
+        return f"mdrs {__version__}"
+
+    def config_create(self, remote: str, url: str) -> None:
+        remote = self.parse_remote_host(remote)
+        config = self.config_class(remote)
+        if config.url is not None:
+            raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
+        else:
+            config.url = url
+
+    def config_update(self, remote: str, url: str) -> None:
+        remote = self.parse_remote_host(remote)
+        config = self.config_class(remote)
+        if config.url is None:
+            raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
+        else:
+            config.url = url
+
+    def config_list(self) -> list:
+        config = self.config_class("")
+        return config.list()
+
+    def config_delete(self, remote: str) -> None:
+        remote = self.parse_remote_host(remote)
+        config = self.config_class(remote)
+        if config.url is None:
+            raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
+        else:
+            del config.url

mdrsclient/commands/__init__.py

@@ -12,6 +12,7 @@ from mdrsclient.commands.mkdir import MkdirCommand
 from mdrsclient.commands.mv import MvCommand
 from mdrsclient.commands.rm import RmCommand
 from mdrsclient.commands.upload import UploadCommand
+from mdrsclient.commands.version import VersionCommand
 from mdrsclient.commands.whoami import WhoamiCommand
 
 __all__ = [
@@ -29,5 +30,6 @@ __all__ = [
     "MvCommand",
     "RmCommand",
     "UploadCommand",
+    "VersionCommand",
     "WhoamiCommand",
 ]

mdrsclient/commands/base.py

@@ -1,18 +1,7 @@
-import re
 from abc import ABC, abstractmethod
 from typing import Any
-from unicodedata import normalize
 
-from mdrsclient.api import FolderApi, LaboratoryApi
+from mdrsclient.exceptions import UnexpectedException
-from mdrsclient.config import ConfigFile
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import (
-    IllegalArgumentException,
-    MissingConfigurationException,
-    UnauthorizedException,
-    UnexpectedException,
-)
-from mdrsclient.models import Folder, Laboratory
 
 
 class BaseCommand(ABC):
@@ -20,64 +9,3 @@ class BaseCommand(ABC):
     @abstractmethod
     def register(cls, parsers: Any) -> None:
         raise UnexpectedException("Not implemented.")
-
-    @classmethod
-    def _create_connection(cls, remote: str) -> MDRSConnection:
-        config = ConfigFile(remote)
-        if config.url is None:
-            raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
-        return MDRSConnection(config.remote, config.url)
-
-    @classmethod
-    def _find_laboratory(cls, connection: MDRSConnection, name: str) -> Laboratory:
-        if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired:
-            laboratory_api = LaboratoryApi(connection)
-            connection.laboratories = laboratory_api.list()
-        laboratory = connection.laboratories.find_by_name(name)
-        if laboratory is None:
-            raise IllegalArgumentException(f"Laboratory `{name}` not found.")
-        return laboratory
-
-    @classmethod
-    def _find_folder(
-        cls, connection: MDRSConnection, laboratory: Laboratory, path: str, password: str | None = None
-    ) -> Folder:
-        folder_api = FolderApi(connection)
-        folders = folder_api.list(laboratory.id, normalize("NFC", path))
-        if len(folders) != 1:
-            raise UnexpectedException(f"Folder `{path}` not found.")
-        if folders[0].lock:
-            if password is None:
-                raise UnauthorizedException(f"Folder `{path}` is locked.")
-            folder_api.auth(folders[0].id, password)
-        return folder_api.retrieve(folders[0].id)
-
-    @classmethod
-    def _parse_remote_host(cls, path: str) -> str:
-        path_array = path.split(":")
-        remote_host = path_array[0]
-        if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2:
-            raise IllegalArgumentException("Invalid remote host")
-        return remote_host
-
-    @classmethod
-    def _parse_remote_host_with_path(cls, path: str) -> tuple[str, str, str]:
-        path = re.sub(r"//+|/\./+|/\.$", "/", path)
-        if re.search(r"/\.\./|/\.\.$", path) is not None:
-            raise IllegalArgumentException("Path traversal found.")
-        path_array = path.split(":")
-        if len(path_array) != 2:
-            raise IllegalArgumentException("Invalid remote host.")
-        remote_host = path_array[0]
-        folder_array = path_array[1].split("/")
-        is_absolute_path = folder_array[0] == ""
-        if not is_absolute_path:
-            raise IllegalArgumentException("Must be absolute paths.")
-        del folder_array[0]
-        if len(folder_array) == 0:
-            laboratory = ""
-            folder = ""
-        else:
-            laboratory = folder_array.pop(0)
-            folder = "/" + "/".join(folder_array)
-        return (remote_host, laboratory, folder)

mdrsclient/commands/chacl.py

@@ -1,7 +1,6 @@
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import FolderApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
 from mdrsclient.models import FolderAccessLevel
@@ -31,10 +30,8 @@ class ChaclCommand(BaseCommand):
 
     @classmethod
     def chacl(cls, remote_path: str, access_level: int, is_recursive: bool, password: str | None) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        r_path = r_path.rstrip("/")
+        from mdrsclient.client import MdrsClient
-        connection = cls._create_connection(remote)
+
-        laboratory = cls._find_laboratory(connection, laboratory_name)
+        client = MdrsClient.from_remote(remote)
-        folder = cls._find_folder(connection, laboratory, r_path)
+        client.chacl(remote_path, access_level, is_recursive, password)
-        folder_api = FolderApi(connection)
-        folder_api.acl(folder.id, access_level, is_recursive, password)

mdrsclient/commands/config.py

@@ -2,8 +2,6 @@ from argparse import Namespace
 from typing import Any, Callable
 
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.config import ConfigFile
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class ConfigCommand(BaseCommand):
@@ -25,11 +23,12 @@ class ConfigCommand(BaseCommand):
         update_parser.add_argument("url", help="API entrypoint url of remote host")
         update_parser.set_defaults(func=cls.func_update)
         # config list
-        list_parser = config_parsers.add_parser("list", help="list all the remote hosts")
+        list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
-        list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
         list_parser.set_defaults(func=cls.func_list)
         # config delete
-        delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host")
+        delete_parser = config_parsers.add_parser(
+            "delete", help="delete an existing remote host", aliases=["remove", "rm"]
+        )
         delete_parser.add_argument("remote", help="label of remote host")
         delete_parser.set_defaults(func=cls.func_delete)
 
@@ -37,56 +36,29 @@ class ConfigCommand(BaseCommand):
     def func_create(cls, args: Namespace) -> None:
         remote = str(args.remote)
         url = str(args.url)
-        cls.create(remote, url)
+        from mdrsclient.client import MdrsClient
+
+        MdrsClient(None).config_create(remote, url)
 
     @classmethod
     def func_update(cls, args: Namespace) -> None:
         remote = str(args.remote)
         url = str(args.url)
-        cls.update(remote, url)
+        from mdrsclient.client import MdrsClient
+
+        MdrsClient(None).config_update(remote, url)
 
     @classmethod
     def func_list(cls, args: Namespace) -> None:
-        is_long = bool(args.long)
+        from mdrsclient.client import MdrsClient
-        cls.list(is_long)
+
+        client = MdrsClient(None)
+        for remote, url in client.config_list():
+            print(f"{remote}:\t{url}")
 
     @classmethod
     def func_delete(cls, args: Namespace) -> None:
         remote = str(args.remote)
-        cls.delete(remote)
+        from mdrsclient.client import MdrsClient
 
-    @classmethod
+        MdrsClient(None).config_delete(remote)
-    def create(cls, remote: str, url: str) -> None:
-        remote = cls._parse_remote_host(remote)
-        config = ConfigFile(remote)
-        if config.url is not None:
-            raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
-        else:
-            config.url = url
-
-    @classmethod
-    def update(cls, remote: str, url: str) -> None:
-        remote = cls._parse_remote_host(remote)
-        config = ConfigFile(remote)
-        if config.url is None:
-            raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
-        else:
-            config.url = url
-
-    @classmethod
-    def list(cls, is_long: bool) -> None:
-        config = ConfigFile("")
-        for remote, url in config.list():
-            line = f"{remote}:"
-            if is_long:
-                line += f"\t{url}"
-            print(line)
-
-    @classmethod
-    def delete(cls, remote: str) -> None:
-        remote = cls._parse_remote_host(remote)
-        config = ConfigFile(remote)
-        if config.url is None:
-            raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
-        else:
-            del config.url

mdrsclient/commands/cp.py

@@ -1,11 +1,7 @@
-import os
 from argparse import Namespace
 from typing import Any
-from unicodedata import normalize
 
-from mdrsclient.api import FileApi, FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class CpCommand(BaseCommand):
@@ -28,51 +24,8 @@ class CpCommand(BaseCommand):
 
     @classmethod
     def cp(cls, src_path: str, dest_path: str, is_recursive: bool) -> None:
-        (s_remote, s_laboratory_name, s_path) = cls._parse_remote_host_with_path(src_path)
+        remote = src_path.split(":", 1)[0] if ":" in src_path else ""
-        (d_remote, d_laboratory_name, d_path) = cls._parse_remote_host_with_path(dest_path)
+        from mdrsclient.client import MdrsClient
-        if s_remote != d_remote:
+
-            raise IllegalArgumentException("Remote host mismatched.")
+        client = MdrsClient.from_remote(remote)
-        if s_laboratory_name != d_laboratory_name:
+        client.cp(src_path, dest_path, is_recursive)
-            raise IllegalArgumentException("Laboratory mismatched.")
-        s_path = s_path.rstrip("/")
-        s_dirname = os.path.dirname(s_path)
-        s_basename = os.path.basename(s_path)
-        if d_path.endswith("/"):
-            d_dirname = d_path
-            d_basename = s_basename
-        else:
-            d_dirname = os.path.dirname(d_path)
-            d_basename = os.path.basename(d_path)
-        connection = cls._create_connection(s_remote)
-        laboratory = cls._find_laboratory(connection, s_laboratory_name)
-        s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
-        d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
-        s_file = s_parent_folder.find_file(s_basename)
-        if s_file is not None:
-            # source is file
-            d_file = d_parent_folder.find_file(d_basename)
-            if d_file is not None:
-                raise IllegalArgumentException(f"File `{d_basename}` already exists.")
-            d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
-            if d_sub_folder is not None:
-                raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
-            file_api = FileApi(connection)
-            if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
-                file_api.copy(s_file, d_parent_folder.id, normalize("NFC", d_basename))
-        else:
-            s_folder = s_parent_folder.find_sub_folder(s_basename)
-            if s_folder is None:
-                raise IllegalArgumentException(f"File or folder `{s_basename}` not found.")
-            # source is folder
-            if not is_recursive:
-                raise IllegalArgumentException(f"Cannot copy `{s_path}`: Is a folder.")
-            if d_parent_folder.find_file(d_basename) is not None:
-                raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
-            d_folder = d_parent_folder.find_sub_folder(d_basename)
-            if d_folder is not None:
-                if d_folder.id == s_folder.id:
-                    raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
-                raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
-            folder_api = FolderApi(connection)
-            if s_parent_folder.id != d_parent_folder.id or s_basename != d_basename:
-                folder_api.copy(s_folder, d_parent_folder.id, normalize("NFC", d_basename))

mdrsclient/commands/download.py

@@ -1,22 +1,7 @@
-import os
 from argparse import Namespace
-from concurrent.futures import ThreadPoolExecutor
 from typing import Any
 
-from pydantic.dataclasses import dataclass
-
-from mdrsclient.api import FileApi, FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import IllegalArgumentException
-from mdrsclient.models import File
-from mdrsclient.settings import CONCURRENT
-
-
-@dataclass(frozen=True)
-class DownloadFileInfo:
-    file: File
-    path: str
 
 
 class DownloadCommand(BaseCommand):
@@ -26,6 +11,15 @@ class DownloadCommand(BaseCommand):
         download_parser.add_argument(
             "-r", "--recursive", help="download folders and their contents recursive", action="store_true"
         )
+        download_parser.add_argument(
+            "-s",
+            "--skip-if-exists",
+            help="skip the download if file is already uploaded and file size is the same",
+            action="store_true",
+        )
+        download_parser.add_argument(
+            "-e", "--exclude", help="exclude to download path matched file or folders", action="append"
+        )
         download_parser.add_argument("-p", "--password", help="password to use when open locked folder")
         download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)")
         download_parser.add_argument("local_path", help="local folder path (/foo/bar/)")
@@ -36,58 +30,24 @@ class DownloadCommand(BaseCommand):
         remote_path = str(args.remote_path)
         local_path = str(args.local_path)
         is_recursive = bool(args.recursive)
+        is_skip_if_exists = bool(args.skip_if_exists)
         password = str(args.password) if args.password else None
-        cls.download(remote_path, local_path, is_recursive, password)
+        excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
+        cls.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
 
     @classmethod
-    def download(cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None) -> None:
+    def download(
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        cls,
-        r_path = r_path.rstrip("/")
+        remote_path: str,
-        r_dirname = os.path.dirname(r_path)
+        local_path: str,
-        r_basename = os.path.basename(r_path)
+        is_recursive: bool,
-        connection = cls._create_connection(remote)
+        is_skip_if_exists: bool,
-        l_dirname = os.path.realpath(local_path)
+        password: str | None,
-        if not os.path.isdir(l_dirname):
+        excludes: list[str],
-            raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
-        laboratory = cls._find_laboratory(connection, laboratory_name)
-        r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
-        file = r_parent_folder.find_file(r_basename)
-        download_files: list[DownloadFileInfo] = []
-        if file is not None:
-            l_path = os.path.join(l_dirname, r_basename)
-            download_files.append(DownloadFileInfo(file, l_path))
-        else:
-            folder = r_parent_folder.find_sub_folder(r_basename)
-            if folder is None:
-                raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
-            if not is_recursive:
-                raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
-            folder_api = FolderApi(connection)
-            cls.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname)
-        cls.__multiple_download(connection, download_files)
-
-    @classmethod
-    def __multiple_download_pickup_recursive_files(
-        cls, folder_api: FolderApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str
     ) -> None:
-        folder = folder_api.retrieve(folder_id)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        dirname = os.path.join(basedir, folder.name)
+        from mdrsclient.client import MdrsClient
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
-        print(dirname)
-        for file in folder.files:
-            path = os.path.join(dirname, file.name)
-            infolist.append(DownloadFileInfo(file, path))
-        for sub_folder in folder.sub_folders:
-            cls.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname)
 
-    @classmethod
+        client = MdrsClient.from_remote(remote)
-    def __multiple_download(cls, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None:
+        client.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
-        file_api = FileApi(connection)
+        return
-        with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_download_worker(file_api, x), infolist)
-
-    @classmethod
-    def __multiple_download_worker(cls, file_api: FileApi, info: DownloadFileInfo) -> None:
-        file_api.download(info.file, info.path)
-        print(info.path)

mdrsclient/commands/file_metadata.py

@@ -1,11 +1,8 @@
 import json
-import os
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import FileApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class FileMetadataCommand(BaseCommand):
@@ -24,16 +21,9 @@ class FileMetadataCommand(BaseCommand):
 
     @classmethod
     def file_metadata(cls, remote_path: str, password: str | None) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        r_path = r_path.rstrip("/")
+        from mdrsclient.client import MdrsClient
-        r_dirname = os.path.dirname(r_path)
+
-        r_basename = os.path.basename(r_path)
+        client = MdrsClient.from_remote(remote)
-        connection = cls._create_connection(remote)
+        metadata = client.file_metadata(remote_path, password)
-        laboratory = cls._find_laboratory(connection, laboratory_name)
-        folder = cls._find_folder(connection, laboratory, r_dirname, password)
-        file = folder.find_file(r_basename)
-        if file is None:
-            raise IllegalArgumentException(f"File `{r_basename}` not found.")
-        file_api = FileApi(connection)
-        metadata = file_api.metadata(file)
         print(json.dumps(metadata, ensure_ascii=False))

mdrsclient/commands/labs.py

@@ -1,7 +1,6 @@
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import LaboratoryApi
 from mdrsclient.commands.base import BaseCommand
 
 
@@ -19,11 +18,11 @@ class LabsCommand(BaseCommand):
 
     @classmethod
     def labs(cls, remote: str) -> None:
-        remote = cls._parse_remote_host(remote)
+        from mdrsclient.client import MdrsClient
-        connection = cls._create_connection(remote)
+
-        laboratory_api = LaboratoryApi(connection)
+        remote_host = MdrsClient.parse_remote_host(remote)
-        laboratories = laboratory_api.list()
+        client = MdrsClient.from_remote(remote_host)
-        connection.laboratories = laboratories
+        laboratories = client.get_laboratories()
         label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
         length: dict[str, int] = {}
         for key in label.keys():
@@ -34,7 +33,6 @@ class LabsCommand(BaseCommand):
             length["pi_name"] = max(length["pi_name"], len(laboratory.pi_name))
             length["full_name"] = max(length["full_name"], len(laboratory.full_name))
         header = (
-            # f"{label['id']:{length['id']}}\t{label['name']:{length['name']}}\t"
             f"{label['name']:{length['name']}}\t"
             f"{label['pi_name']:{length['pi_name']}}\t{label['full_name']:{length['full_name']}}"
         )
@@ -42,7 +40,6 @@ class LabsCommand(BaseCommand):
         print("-" * len(header.expandtabs()))
         for laboratory in laboratories:
             print(
-                # f"{laboratory.id:{length['id']}}\t{laboratory.name:{length['name']}}\t"
                 f"{laboratory.name:{length['name']}}\t"
                 f"{laboratory.pi_name:{length['pi_name']}}\t{laboratory.full_name:{length['full_name']}}"
             )

mdrsclient/commands/login.py

@@ -2,36 +2,30 @@ import getpass
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import UserApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.config import ConfigFile
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import MissingConfigurationException
 
 
 class LoginCommand(BaseCommand):
     @classmethod
     def register(cls, parsers: Any) -> None:
         login_parser = parsers.add_parser("login", help="login to remote host")
+        login_parser.add_argument("-u", "--username", help="login username")
+        login_parser.add_argument("-p", "--password", help="login password")
         login_parser.add_argument("remote", help="label of remote host")
         login_parser.set_defaults(func=cls.func)
 
     @classmethod
     def func(cls, args: Namespace) -> None:
         remote = str(args.remote)
-        username = input("Username: ").strip()
+        username = str(args.username) if args.username else input("Username: ").strip()
-        password = getpass.getpass("Password: ").strip()
+        password = str(args.password) if args.password else getpass.getpass("Password: ").strip()
         cls.login(remote, username, password)
 
     @classmethod
     def login(cls, remote: str, username: str, password: str) -> None:
-        remote = cls._parse_remote_host(remote)
+        from mdrsclient.client import MdrsClient
-        config = ConfigFile(remote)
+
-        if config.url is None:
+        remote_host = MdrsClient.parse_remote_host(remote)
-            raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
+        client = MdrsClient.from_remote(remote_host)
-        connection = MDRSConnection(config.remote, config.url)
+        client.login(username, password)
-        user_api = UserApi(connection)
-        (user, token) = user_api.auth(username, password)
         print("Login Successful")
-        connection.user = user
-        connection.token = token

mdrsclient/commands/logout.py

@@ -2,9 +2,6 @@ from argparse import Namespace
 from typing import Any
 
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.config import ConfigFile
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import MissingConfigurationException
 
 
 class LogoutCommand(BaseCommand):
@@ -21,9 +18,8 @@ class LogoutCommand(BaseCommand):
 
     @classmethod
     def logout(cls, remote: str) -> None:
-        remote = cls._parse_remote_host(remote)
+        from mdrsclient.client import MdrsClient
-        config = ConfigFile(remote)
+
-        if config.url is None:
+        remote_host = MdrsClient.parse_remote_host(remote)
-            raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
+        client = MdrsClient.from_remote(remote_host)
-        connection = MDRSConnection(config.remote, config.url)
+        client.logout()
-        connection.logout()

mdrsclient/commands/ls.py

@@ -1,12 +1,13 @@
 import json
 from argparse import Namespace
-from typing import Any
+from typing import Any, Final
 
 from pydantic.dataclasses import dataclass
 
-from mdrsclient.api import FolderApi
+from mdrsclient.api import FoldersApi
+from mdrsclient.client import MdrsClient
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.connection import MDRSConnection
+from mdrsclient.config import build_download_url
 from mdrsclient.exceptions import UnauthorizedException
 from mdrsclient.models import File, Folder, FolderSimple, Laboratory
 
@@ -19,11 +20,11 @@ class Config:
 @dataclass(config=Config)
 class LsCommandContext:
     prefix: str
-    connection: MDRSConnection
+    client: MdrsClient
     laboratory: Laboratory
     password: str
     is_json: bool
-    is_quick: bool
+    is_quiet: bool
     is_recursive: bool
 
 
@@ -35,7 +36,7 @@ class LsCommand(BaseCommand):
         ls_parser.add_argument("-J", "--json", help="turn on json output", action="store_true")
         ls_parser.add_argument(
             "-q",
-            "--quick",
+            "--quiet",
             help="don't output header row. this option is forced if the -r option is specified",
             action="store_true",
         )
@@ -49,60 +50,98 @@ class LsCommand(BaseCommand):
         password = str(args.password) if args.password else None
         is_json = bool(args.json)
         is_recursive = bool(args.recursive)
-        is_quick = bool(args.quick) if not is_recursive else True
+        is_quiet = bool(args.quiet) if not is_recursive else True
-        cls.ls(remote_path, password, is_json, is_recursive, is_quick)
+        cls.ls(remote_path, password, is_json, is_recursive, is_quiet)
 
     @classmethod
-    def ls(cls, remote_path: str, password: str | None, is_json: bool, is_recursive: bool, is_quick: bool) -> None:
+    def ls(cls, remote_path: str, password: str | None, is_json: bool, is_recursive: bool, is_quiet: bool) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        connection = cls._create_connection(remote)
+        from mdrsclient.client import MdrsClient
-        laboratory = cls._find_laboratory(connection, laboratory_name)
+
+        client = MdrsClient.from_remote(remote)
+        cls._ls_logic(client, remote_path, password, is_json, is_recursive, is_quiet)
+        return
+
+    @classmethod
+    def _ls_logic(
+        cls,
+        client: MdrsClient,
+        remote_path: str,
+        password: str | None,
+        is_json: bool,
+        is_recursive: bool,
+        is_quiet: bool,
+    ) -> None:
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
+
+        folder, laboratory = client.resolve_folder(remote_path, password)
+        laboratory_name = laboratory.name
+        files = client.find_files(folder.id)
         context = LsCommandContext(
             f"{remote}:/{laboratory_name}",
-            connection,
+            client,
             laboratory,
             password if password is not None else "",
             is_json,
-            is_quick,
+            is_quiet,
             is_recursive,
         )
-        folder = cls._find_folder(connection, laboratory, r_path, password)
         if context.is_json:
-            cls._ls_json(context, folder)
+            cls._ls_json(context, folder, files)
         else:
-            cls._ls_plain(context, folder)
+            cls._ls_plain(context, folder, files)
 
     @classmethod
-    def _ls_json(cls, context: LsCommandContext, folder: Folder) -> None:
+    def _ls_json(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
-        print(json.dumps(cls._folder2dict(context, folder), ensure_ascii=False))
+        print(json.dumps(cls._folder2dict(context, folder, files), ensure_ascii=False))
 
-    @classmethod
+    LABELS: Final[dict[str, str]] = {
-    def _ls_plain(cls, context: LsCommandContext, folder: Folder) -> None:
-        label = {
         "type": "Type",
         "acl": "Access",
         "laboratory": "Laboratory",
-            "size": "Lock/Size",
+        "size": "Size",
         "date": "Date",
         "name": "Name",
     }
-        length: dict[str, int] = {}
+
-        for key in label.keys():
+    @classmethod
-            length[key] = len(label[key]) if not context.is_quick else 0
+    def _column_widths(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> dict[str, int]:
+        """Width of each column: the widest of its heading and everything printed under it."""
+        length = {key: len(label) if not context.is_quiet else 0 for key, label in cls.LABELS.items()}
         for sub_folder in folder.sub_folders:
-            sub_laboratory = context.connection.laboratories.find_by_id(sub_folder.lab_id)
+            sub_laboratory_name = cls._laboratory_name(context, sub_folder.laboratory_id)
-            sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
             length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
             length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
-            length["size"] = max(length["size"], len(sub_folder.lock_name))
+            length["size"] = max(length["size"], len(str(sub_folder.size)))
             length["date"] = max(length["date"], len(sub_folder.updated_at_name))
             length["name"] = max(length["name"], len(sub_folder.name))
-        for file in folder.files:
+        for file in files:
             length["size"] = max(length["size"], len(str(file.size)))
             length["date"] = max(length["date"], len(file.updated_at_name))
             length["name"] = max(length["name"], len(file.name))
         length["acl"] = max(length["acl"], len(folder.access_level_name))
         length["laboratory"] = max(length["laboratory"], len(context.laboratory.name))
+        return length
+
+    @classmethod
+    def _ls_plain_children(cls, context: LsCommandContext, folder: Folder) -> None:
+        """List each sub-folder in turn, passing over the ones the caller cannot open."""
+        print("")
+        folder_api = FoldersApi(context.client.connection)
+        for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
+            try:
+                if sub_folder.lock:
+                    folder_api.auth(sub_folder.id, context.password)
+                sub_detail = folder_api.retrieve(sub_folder.id)
+                sub_files = context.client.find_files(sub_folder.id)
+            except UnauthorizedException:
+                continue
+            cls._ls_plain(context, sub_detail, sub_files)
+
+    @classmethod
+    def _ls_plain(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
+        label = cls.LABELS
+        length = cls._column_widths(context, folder, files)
         header = (
             f"{label['type']:{length['type']}}\t{label['acl']:{length['acl']}}\t"
             f"{label['laboratory']:{length['laboratory']}}\t{label['size']:{length['size']}}\t"
@@ -111,20 +150,21 @@ class LsCommand(BaseCommand):
 
         if context.is_recursive:
             print(f"{context.prefix}{folder.path}:")
-            print(f"total {sum(f.size for f in folder.files)}")
+            print(f"total {sum(f.size for f in files)}")
 
-        if not context.is_quick:
+        if not context.is_quiet:
             print(header)
             print("-" * len(header.expandtabs()))
 
         for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
-            sub_laboratory_name = cls._laboratory_name(context, sub_folder.lab_id)
+            sub_laboratory_name = cls._laboratory_name(context, sub_folder.laboratory_id)
+            sub_folder_type = "[d]" if sub_folder.lock is False else "[l]"
             print(
-                f"{'[d]':{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
+                f"{sub_folder_type:{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
-                f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.lock_name:{length['size']}}\t"
+                f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.size:{length['size']}}\t"
                 f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
             )
-        for file in sorted(folder.files, key=lambda x: x.name):
+        for file in sorted(files, key=lambda x: x.name):
             print(
                 f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t"
                 f"{context.laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t"
@@ -132,32 +172,26 @@ class LsCommand(BaseCommand):
             )
 
         if context.is_recursive:
-            print("")
+            cls._ls_plain_children(context, folder)
-            for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
-                folder_api = FolderApi(context.connection)
-                try:
-                    if sub_folder.lock:
-                        folder_api.auth(sub_folder.id, context.password)
-                    folder = folder_api.retrieve(sub_folder.id)
-                    cls._ls_plain(context, folder)
-                except UnauthorizedException:
-                    pass
 
     @classmethod
-    def _folder2dict(cls, context: LsCommandContext, folder: Folder | FolderSimple) -> dict[str, Any]:
+    def _folder2dict(
+        cls, context: LsCommandContext, folder: Folder | FolderSimple, files: list[File]
+    ) -> dict[str, Any]:
         data: dict[str, Any] = {
             "id": folder.id,
             "pid": folder.pid,
             "name": folder.name,
+            "size": folder.size,
             "access_level": folder.access_level_name,
             "lock": folder.lock,
-            "laboratory": cls._laboratory_name(context, folder.lab_id),
+            "laboratory": cls._laboratory_name(context, folder.laboratory_id),
             "description": folder.description,
             "created_at": folder.created_at,
             "updated_at": folder.updated_at,
         }
         if isinstance(folder, Folder):
-            folder_api = FolderApi(context.connection)
+            folder_api = FoldersApi(context.client.connection)
             data["metadata"] = folder_api.metadata(folder.id)
             if context.is_recursive:
                 sub_folders: list[dict[str, Any]] = []
@@ -166,15 +200,16 @@ class LsCommand(BaseCommand):
                         if sub_folder.lock:
                             folder_api.auth(sub_folder.id, context.password)
                         folder2 = folder_api.retrieve(sub_folder.id)
-                        sub_folders.append(cls._folder2dict(context, folder2))
+                        files2 = context.client.find_files(sub_folder.id)
+                        sub_folders.append(cls._folder2dict(context, folder2, files2))
                     except UnauthorizedException:
                         pass
                 data["sub_folders"] = sub_folders
             else:
                 data["sub_folders"] = list(
-                    map(lambda x: cls._folder2dict(context, x), sorted(folder.sub_folders, key=lambda x: x.name))
+                    map(lambda x: cls._folder2dict(context, x, []), sorted(folder.sub_folders, key=lambda x: x.name))
                 )
-            data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(folder.files, key=lambda x: x.name)))
+            data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(files, key=lambda x: x.name)))
         return data
 
     @classmethod
@@ -187,7 +222,7 @@ class LsCommand(BaseCommand):
             # "thumbnail": file.thumbnail,
             "description": file.description,
             "metadata": file.metadata,
-            "download_url": f"{context.connection.url}/v2/{file.download_url}",
+            "download_url": build_download_url(context.client.connection.url, file.download_url),
             "created_at": file.created_at,
             "updated_at": file.updated_at,
         }
@@ -195,5 +230,5 @@ class LsCommand(BaseCommand):
 
     @classmethod
     def _laboratory_name(cls, context: LsCommandContext, laboratory_id: int) -> str:
-        laboratory = context.connection.laboratories.find_by_id(laboratory_id)
+        laboratory = context.client.connection.laboratories.find_by_id(laboratory_id)
         return laboratory.name if laboratory is not None else "(invalid)"

mdrsclient/commands/metadata.py

@@ -2,7 +2,6 @@ import json
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import FolderApi
 from mdrsclient.commands.base import BaseCommand
 
 
@@ -22,10 +21,9 @@ class MetadataCommand(BaseCommand):
 
     @classmethod
     def metadata(cls, remote_path: str, password: str | None) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        connection = cls._create_connection(remote)
+        from mdrsclient.client import MdrsClient
-        laboratory = cls._find_laboratory(connection, laboratory_name)
+
-        folder = cls._find_folder(connection, laboratory, r_path, password)
+        client = MdrsClient.from_remote(remote)
-        folder_api = FolderApi(connection)
+        metadata = client.metadata(remote_path, password)
-        metadata = folder_api.metadata(folder.id)
         print(json.dumps(metadata, ensure_ascii=False))

mdrsclient/commands/mkdir.py

@@ -1,11 +1,7 @@
-import os
 from argparse import Namespace
 from typing import Any
-from unicodedata import normalize
 
-from mdrsclient.api import FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class MkdirCommand(BaseCommand):
@@ -22,14 +18,8 @@ class MkdirCommand(BaseCommand):
 
     @classmethod
     def mkdir(cls, remote_path: str) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        r_path = r_path.rstrip("/")
+        from mdrsclient.client import MdrsClient
-        r_dirname = os.path.dirname(r_path)
+
-        r_basename = os.path.basename(r_path)
+        client = MdrsClient.from_remote(remote)
-        connection = cls._create_connection(remote)
+        client.mkdir(remote_path)
-        laboratory = cls._find_laboratory(connection, laboratory_name)
-        parent_folder = cls._find_folder(connection, laboratory, r_dirname)
-        if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None:
-            raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
-        folder_api = FolderApi(connection)
-        folder_api.create(normalize("NFC", r_basename), parent_folder.id)

mdrsclient/commands/mv.py

@@ -1,11 +1,7 @@
-import os
 from argparse import Namespace
 from typing import Any
-from unicodedata import normalize
 
-from mdrsclient.api import FileApi, FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class MvCommand(BaseCommand):
@@ -24,49 +20,8 @@ class MvCommand(BaseCommand):
 
     @classmethod
     def mv(cls, src_path: str, dest_path: str) -> None:
-        (s_remote, s_laboratory_name, s_path) = cls._parse_remote_host_with_path(src_path)
+        remote = src_path.split(":", 1)[0] if ":" in src_path else ""
-        (d_remote, d_laboratory_name, d_path) = cls._parse_remote_host_with_path(dest_path)
+        from mdrsclient.client import MdrsClient
-        if s_remote != d_remote:
+
-            raise IllegalArgumentException("Remote host mismatched.")
+        client = MdrsClient.from_remote(remote)
-        if s_laboratory_name != d_laboratory_name:
+        client.mv(src_path, dest_path)
-            raise IllegalArgumentException("Laboratory mismatched.")
-        s_path = s_path.rstrip("/")
-        s_dirname = os.path.dirname(s_path)
-        s_basename = os.path.basename(s_path)
-        if d_path.endswith("/"):
-            d_dirname = d_path
-            d_basename = s_basename
-        else:
-            d_dirname = os.path.dirname(d_path)
-            d_basename = os.path.basename(d_path)
-        connection = cls._create_connection(s_remote)
-        laboratory = cls._find_laboratory(connection, s_laboratory_name)
-        s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
-        d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
-        s_file = s_parent_folder.find_file(s_basename)
-        if s_file is not None:
-            # source is file
-            d_file = d_parent_folder.find_file(d_basename)
-            if d_file is not None:
-                raise IllegalArgumentException(f"File `{d_basename}` already exists.")
-            d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
-            if d_sub_folder is not None:
-                raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
-            file_api = FileApi(connection)
-            if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
-                file_api.move(s_file, d_parent_folder.id, normalize("NFC", d_basename))
-        else:
-            s_folder = s_parent_folder.find_sub_folder(s_basename)
-            if s_folder is None:
-                raise IllegalArgumentException(f"File or folder `{s_basename}` not found.")
-            # source is folder
-            if d_parent_folder.find_file(d_basename) is not None:
-                raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
-            d_folder = d_parent_folder.find_sub_folder(d_basename)
-            if d_folder is not None:
-                if d_folder.id == s_folder.id:
-                    raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
-                raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
-            folder_api = FolderApi(connection)
-            if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
-                folder_api.move(s_folder, d_parent_folder.id, normalize("NFC", d_basename))

mdrsclient/commands/rm.py

@@ -1,10 +1,7 @@
-import os
 from argparse import Namespace
 from typing import Any
 
-from mdrsclient.api import FileApi, FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.exceptions import IllegalArgumentException
 
 
 class RmCommand(BaseCommand):
@@ -25,22 +22,8 @@ class RmCommand(BaseCommand):
 
     @classmethod
     def rm(cls, remote_path: str, is_recursive: bool) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        r_path = r_path.rstrip("/")
+        from mdrsclient.client import MdrsClient
-        r_dirname = os.path.dirname(r_path)
+
-        r_basename = os.path.basename(r_path)
+        client = MdrsClient.from_remote(remote)
-        connection = cls._create_connection(remote)
+        client.rm(remote_path, is_recursive)
-        laboratory = cls._find_laboratory(connection, laboratory_name)
-        parent_folder = cls._find_folder(connection, laboratory, r_dirname)
-        file = parent_folder.find_file(r_basename)
-        if file is not None:
-            file_api = FileApi(connection)
-            file_api.destroy(file)
-        else:
-            folder = parent_folder.find_sub_folder(r_basename)
-            if folder is None:
-                raise IllegalArgumentException(f"Cannot remove `{r_path}`: No such file or folder.")
-            if not is_recursive:
-                raise IllegalArgumentException(f"Cannot remove `{r_path}`: Is a folder.")
-            folder_api = FolderApi(connection)
-            folder_api.destroy(folder.id, True)

mdrsclient/commands/upload.py

@@ -1,22 +1,7 @@
-import os
 from argparse import Namespace
-from concurrent.futures import ThreadPoolExecutor
 from typing import Any
 
-from pydantic.dataclasses import dataclass
-
-from mdrsclient.api import FileApi, FolderApi
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import IllegalArgumentException, MDRSException
-from mdrsclient.models import Folder
-from mdrsclient.settings import CONCURRENT
-
-
-@dataclass(frozen=True)
-class UploadFileInfo:
-    folder: Folder
-    path: str
 
 
 class UploadCommand(BaseCommand):
@@ -26,6 +11,12 @@ class UploadCommand(BaseCommand):
         upload_parser.add_argument(
             "-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
         )
+        upload_parser.add_argument(
+            "-s",
+            "--skip-if-exists",
+            help="skip the upload if file is already uploaded and file size is the same",
+            action="store_true",
+        )
         upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
         upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
         upload_parser.set_defaults(func=cls.func)
@@ -35,66 +26,14 @@ class UploadCommand(BaseCommand):
         local_path = str(args.local_path)
         remote_path = str(args.remote_path)
         is_recursive = bool(args.recursive)
-        cls.upload(local_path, remote_path, is_recursive)
+        is_skip_if_exists = bool(args.skip_if_exists)
+        cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
 
     @classmethod
-    def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
+    def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
-        (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
+        remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
-        l_path = os.path.realpath(local_path)
+        from mdrsclient.client import MdrsClient
-        if not os.path.exists(l_path):
-            raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
-        connection = cls._create_connection(remote)
-        laboratory = cls._find_laboratory(connection, laboratory_name)
-        folder = cls._find_folder(connection, laboratory, r_path)
-        infos: list[UploadFileInfo] = []
-        if os.path.isdir(l_path):
-            if not is_recursive:
-                raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
-            folder_api = FolderApi(connection)
-            folder_map: dict[str, Folder] = {}
-            folder_map[r_path] = folder
-            l_basename = os.path.basename(l_path)
-            for dirpath, _, filenames in os.walk(l_path):
-                sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
-                d_dirname = os.path.join(r_path, sub)
-                d_basename = os.path.basename(d_dirname)
-                # prepare destination parent path
-                d_parent_dirname = os.path.dirname(d_dirname)
-                if folder_map.get(d_parent_dirname) is None:
-                    folder_map[d_parent_dirname] = cls._find_folder(connection, laboratory, d_parent_dirname)
-                # prepare destination path
-                if folder_map.get(d_dirname) is None:
-                    d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
-                    if d_folder is None:
-                        d_folder_id = folder_api.create(d_basename, folder_map[d_parent_dirname].id)
-                    else:
-                        d_folder_id = d_folder.id
-                    print(d_dirname)
-                    folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
-                    if d_folder is None:
-                        folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
-                # register upload file list
-                for filename in filenames:
-                    infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
-        else:
-            infos.append(UploadFileInfo(folder, l_path))
-        cls.__multiple_upload(connection, infos)
 
-    @classmethod
+        client = MdrsClient.from_remote(remote)
-    def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
+        client.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
-        file_api = FileApi(connection)
+        return
-        with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
-
-    @classmethod
-    def __multiple_upload_worker(cls, file_api: FileApi, info: UploadFileInfo) -> None:
-        basename = os.path.basename(info.path)
-        file = info.folder.find_file(basename)
-        try:
-            if file is None:
-                file_api.create(info.folder.id, info.path)
-            else:
-                file_api.update(file, info.path)
-            print(os.path.join(info.folder.path, basename))
-        except MDRSException as e:
-            print(f"API Error: {e}")

mdrsclient/commands/version.py

@@ -0,0 +1,23 @@
+from argparse import Namespace
+from typing import Any
+
+from mdrsclient.commands.base import BaseCommand
+
+
+class VersionCommand(BaseCommand):
+    @classmethod
+    def register(cls, parsers: Any) -> None:
+        version_parser = parsers.add_parser("version", help="show the version of this tool")
+        version_parser.set_defaults(func=cls.func)
+
+    @classmethod
+    def func(cls, args: Namespace) -> None:
+        cls.version()
+
+    @classmethod
+    def version(cls) -> None:
+        from mdrsclient.client import MdrsClient
+
+        # Client initialization is not strictly needed for version, but for consistency:
+        client = MdrsClient(None)
+        print(client.version())

mdrsclient/commands/whoami.py

@@ -2,9 +2,6 @@ from argparse import Namespace
 from typing import Any, Final
 
 from mdrsclient.commands.base import BaseCommand
-from mdrsclient.config import ConfigFile
-from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import MissingConfigurationException
 
 
 class WhoamiCommand(BaseCommand):
@@ -23,12 +20,15 @@ class WhoamiCommand(BaseCommand):
 
     @classmethod
     def whoami(cls, remote: str) -> None:
-        remote = cls._parse_remote_host(remote)
+        from mdrsclient.client import MdrsClient
-        config = ConfigFile(remote)
+
-        if config.url is None:
+        remote_host = MdrsClient.parse_remote_host(remote)
-            raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
+        client = MdrsClient.from_remote(remote_host)
-        connection = MDRSConnection(config.remote, config.url)
+        if client.connection.token is not None and client.connection.token.is_expired:
-        if connection.token is not None and connection.token.is_expired:
+            client.logout()
-            connection.logout()
+        try:
-        username = connection.user.username if connection.user is not None else cls.ANONYMOUS_USERNAME
+            user = client.whoami()
+            username = user.username if user is not None else cls.ANONYMOUS_USERNAME
+        except Exception:
+            username = cls.ANONYMOUS_USERNAME
         print(username)

mdrsclient/config.py

@@ -1,15 +1,97 @@
 import configparser
 import os
-from typing import Final
+import threading
+from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
 
-import validators  # type: ignore
+import validators
 
 from mdrsclient.exceptions import IllegalArgumentException
 from mdrsclient.settings import CONFIG_DIRNAME
 from mdrsclient.utils import FileLock
 
 
-class ConfigFile:
+def normalize_url(url: str) -> str:
+    """
+    Check a remote URL and put it in the one form every client agrees on.
+
+    `simple_host` is what lets a bare hostname through, so a development server on
+    `localhost` is as acceptable as a deployment behind a domain name. The trailing
+    slash goes because the URL is joined with a path that brings its own.
+    """
+    if not validators.url(url, simple_host=True, validate_scheme=lambda scheme: scheme in ("http", "https")):
+        raise IllegalArgumentException("malformed URI sequence")
+    return url.rstrip("/")
+
+
+def build_download_url(base_url: str | None, path: str) -> str:
+    """
+    Join a remote base URL with a path the API returned.
+
+    The API answers with a relative path and no leading separator, and a configuration
+    written before the URL was normalised may still carry a trailing one.
+    """
+    if path.startswith(("http://", "https://")):
+        return path
+    return f"{(base_url or '').rstrip('/')}/{path.lstrip('/')}"
+
+
+@runtime_checkable
+class ConfigInterface(Protocol):
+    remote: str
+
+    if TYPE_CHECKING:
+        # Declared for the type checker, because the class itself is passed around and
+        # called: without this it describes something that takes no arguments to build.
+        # Kept out of the running class, because a protocol that defines `__init__` loses
+        # the guard that stops it being instantiated, and hands that empty `__init__` to
+        # any implementation that does not write its own.
+        def __init__(self, remote: str) -> None: ...
+
+    def list(self) -> list[tuple[str, str]]: ...
+    @property
+    def url(self) -> str | None: ...
+    @url.setter
+    def url(self, url: str) -> None: ...
+    @url.deleter
+    def url(self) -> None: ...
+
+
+class InMemoryConfig(ConfigInterface):
+    __configs: dict[str, str] = {}
+    __lock: threading.Lock = threading.Lock()
+    remote: str
+
+    def __init__(self, remote: str) -> None:
+        self.remote = remote
+
+    def list(self) -> list[tuple[str, str]]:
+        with self.__lock:
+            return list(self.__configs.items())
+
+    @property
+    def url(self) -> str | None:
+        with self.__lock:
+            return self.__configs.get(self.remote)
+
+    @url.setter
+    def url(self, url: str) -> None:
+        url = normalize_url(url)
+        with self.__lock:
+            self.__configs[self.remote] = url
+
+    @url.deleter
+    def url(self) -> None:
+        with self.__lock:
+            if self.remote in self.__configs:
+                del self.__configs[self.remote]
+
+    @classmethod
+    def clear(cls) -> None:
+        with cls.__lock:
+            cls.__configs.clear()
+
+
+class ConfigFile(ConfigInterface):
     OPTION_URL: Final[str] = "url"
     CONFIG_FILENAME: Final[str] = "config.ini"
     remote: str
@@ -41,8 +123,7 @@ class ConfigFile:
 
     @url.setter
     def url(self, url: str) -> None:
-        if not validators.url(url):  # type: ignore
+        url = normalize_url(url)
-            raise IllegalArgumentException("malformed URI sequence")
         self.__load()
         if self.__config.has_section(self.remote):
             self.__config.remove_section(self.remote)

mdrsclient/connection.py

@@ -1,15 +1,16 @@
 import platform
 import threading
-from io import BufferedReader
+from contextlib import AbstractContextManager
 from typing import TypedDict
 
 from requests import Response, Session
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 # Unpack is new in 3.11
 from typing_extensions import Unpack
 
 from mdrsclient.__version__ import __version__
-from mdrsclient.cache import CacheFile
+from mdrsclient.cache import CacheFile, CacheInterface
 from mdrsclient.exceptions import MissingConfigurationException
 from mdrsclient.models import Laboratories, Token, User
 
@@ -21,14 +22,15 @@ class _KwArgsMDRSConnectionGet(TypedDict, total=False):
 
 class _KwArgsMDRSConnectionPost(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
+    timeout: float | tuple[float, float]
 
 
 class _KwArgsMDRSConnectionPut(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
 
 
 class _KwArgsMDRSConnectionDelete(TypedDict, total=False):
@@ -39,14 +41,14 @@ class MDRSConnection:
     url: str
     session: Session
     lock: threading.Lock
-    __cache: CacheFile
+    __cache: CacheInterface
 
-    def __init__(self, remote: str, url: str) -> None:
+    def __init__(self, remote: str, url: str, cache: CacheInterface | None = None) -> None:
         super().__init__()
         self.url = url
         self.session = Session()
         self.lock = threading.Lock()
-        self.__cache = CacheFile(remote)
+        self.__cache = cache if cache is not None else CacheFile(remote)
         self.__prepare_headers()
 
     def get(self, url: str, **kwargs: Unpack[_KwArgsMDRSConnectionGet]) -> Response:
@@ -61,6 +63,14 @@ class MDRSConnection:
     def delete(self, url: str, **kwargs: Unpack[_KwArgsMDRSConnectionDelete]) -> Response:
         return self.session.delete(self.__build_url(url), **kwargs)
 
+    def cache_lock(self) -> AbstractContextManager[None]:
+        """Hold exclusive access to the login cache across every process using it."""
+        return self.__cache.lock()
+
+    def reload_cache(self) -> None:
+        """Re-read the login cache, discarding anything held from an earlier read."""
+        self.__cache.reload()
+
     def logout(self) -> None:
         del self.__cache.user
         del self.__cache.token
@@ -92,9 +102,13 @@ class MDRSConnection:
         self.__cache.laboratories = laboratories
 
     def __build_url(self, path: str) -> str:
+        if path.startswith("http://") or path.startswith("https://"):
+            return path
         if self.url == "":
             raise MissingConfigurationException("remote host is not configured")
-        return f"{self.url}/{path}"
+        # The path brings its own separator, and a configuration written before the URL
+        # was normalised may still carry a trailing slash of its own.
+        return f"{self.url.rstrip('/')}/{path}"
 
     def __prepare_headers(self) -> None:
         self.session.headers.update(

mdrsclient/exceptions.py

@@ -28,6 +28,18 @@ class UnauthorizedException(MDRSException):
     pass
 
 
+class TokenExpiredException(UnauthorizedException):
+    """Thrown when the access token was still valid when sent but had expired on arrival"""
+
+    pass
+
+
+class ServerBusyException(MDRSException):
+    """Thrown when the server did not start handling a request before its token lapsed"""
+
+    pass
+
+
 class ForbiddenException(MDRSException):
     """Thrown when the current user does not have enough privileges to access the resource"""
 

mdrsclient/models/__init__.py

@@ -1,3 +1,4 @@
+from mdrsclient.models.doi import Doi
 from mdrsclient.models.error import DRFStandardizedErrors
 from mdrsclient.models.file import File
 from mdrsclient.models.folder import Folder, FolderAccessLevel, FolderSimple
@@ -6,6 +7,7 @@ from mdrsclient.models.user import Token, User
 
 __all__ = [
     "DRFStandardizedErrors",
+    "Doi",
     "File",
     "Folder",
     "FolderAccessLevel",

mdrsclient/models/doi.py

@@ -0,0 +1,11 @@
+from pydantic.dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Doi:
+    """Model representing a DOI entity (Response from GET v3/doi/{id}/)."""
+
+    # The internal DOI suffix ID returned as a string (e.g. "20260429-001").
+    id: str
+    doi: str
+    folder_id: str

mdrsclient/models/file.py

@@ -1,4 +1,5 @@
 from typing import Any
+from unicodedata import normalize
 
 from pydantic.dataclasses import dataclass
 
@@ -25,3 +26,8 @@ class File:
     @property
     def updated_at_name(self) -> str:
         return iso8601_to_user_friendly(self.updated_at)
+
+
+def find_file(files: list[File], name: str) -> File | None:
+    _name = normalize("NFC", name).lower()
+    return next((x for x in files if x.name.lower() == _name), None)

mdrsclient/models/folder.py

@@ -3,33 +3,41 @@ from unicodedata import normalize
 
 from pydantic.dataclasses import dataclass
 
-from mdrsclient.models.file import File
 from mdrsclient.models.utils import iso8601_to_user_friendly
 
 
 class FolderAccessLevelItem(NamedTuple):
-    id: int
+    mask: int
     key: str
     label: str
 
 
 class FolderAccessLevel:
+    # Bit Mask
+    # - bit 0: Is Private
+    # - bit 1: Is Public
+    # - bit 2: With Password
+    # - bit 3-7: (Reserved)
+    # - bit 8-15: Restricted Open
     ACCESS_LEVELS: Final[list[FolderAccessLevelItem]] = [
-        FolderAccessLevelItem(-1, "storage", "Storage"),
+        FolderAccessLevelItem(0x0204, "5kikan_or_pw_open", "5Kikan or PW Open"),
-        FolderAccessLevelItem(0, "private", "Private"),
+        FolderAccessLevelItem(0x0104, "cbs_or_pw_open", "CBS or PW Open"),
-        FolderAccessLevelItem(1, "cbs_open", "CBS Open"),
+        FolderAccessLevelItem(0x0200, "5kikan_open", "5Kikan Open"),
-        FolderAccessLevelItem(2, "pw_open", "PW Open"),
+        FolderAccessLevelItem(0x0100, "cbs_open", "CBS Open"),
-        FolderAccessLevelItem(3, "public", "Public"),
+        FolderAccessLevelItem(0x0004, "pw_open", "PW Open"),
+        FolderAccessLevelItem(0x0002, "public", "Public"),
+        FolderAccessLevelItem(0x0001, "private", "Private"),
+        FolderAccessLevelItem(0x0000, "storage", "Storage"),
     ]
 
     @staticmethod
     def key2id(key: str) -> int | None:
         acl = next((x for x in FolderAccessLevel.ACCESS_LEVELS if x.key == key), None)
-        return acl.id if acl is not None else None
+        return acl.mask if acl is not None else None
 
     @staticmethod
     def id2label(id: int) -> str | None:
-        acl = next((x for x in FolderAccessLevel.ACCESS_LEVELS if x.id == id), None)
+        acl = next((x for x in FolderAccessLevel.ACCESS_LEVELS if (x.mask & id) == x.mask), None)
         return acl.label if acl is not None else None
 
 
@@ -40,7 +48,8 @@ class FolderSimple:
     name: str
     access_level: int
     lock: bool
-    lab_id: int
+    size: int
+    laboratory_id: int
     description: str
     created_at: str
     updated_at: str
@@ -68,13 +77,8 @@ class FolderSimple:
 class Folder(FolderSimple):
     metadata: list[dict[str, Any]]
     sub_folders: list[FolderSimple]
-    files: list[File]
     path: str
 
     def find_sub_folder(self, name: str) -> FolderSimple | None:
         _name = normalize("NFC", name).lower()
         return next((x for x in self.sub_folders if x.name.lower() == _name), None)
-
-    def find_file(self, name: str) -> File | None:
-        _name = normalize("NFC", name).lower()
-        return next((x for x in self.files if x.name.lower() == _name), None)

mdrsclient/services.py

@@ -0,0 +1,251 @@
+import os
+import re
+from unicodedata import normalize
+
+from mdrsclient.api import DoiApi, FilesApi, FoldersApi, LaboratoriesApi, UsersApi
+from mdrsclient.cache import CacheInterface
+from mdrsclient.config import ConfigFile, ConfigInterface
+from mdrsclient.connection import MDRSConnection
+from mdrsclient.exceptions import (
+    IllegalArgumentException,
+    MissingConfigurationException,
+    UnauthorizedException,
+    UnexpectedException,
+)
+from mdrsclient.models import File, Folder, Laboratory, Token, User
+from mdrsclient.utils import page_num_from_url
+
+
+class MdrsService:
+    config_class: type[ConfigInterface] = ConfigFile
+
+    def __init__(self, connection: MDRSConnection | None, config_class: type[ConfigInterface] | None = None):
+        self.__connection = connection
+        if config_class is not None:
+            self.config_class = config_class
+
+    @property
+    def connection(self) -> MDRSConnection:
+        """
+        The connection every remote operation goes through.
+
+        Optional to supply, because `config` and `version` do their work without a remote
+        and are reached through the same client. Asking for it when none was given is a
+        mistake in the caller, and says so rather than failing later on `None`.
+        """
+        if self.__connection is None:
+            raise MissingConfigurationException("This operation requires a remote host.")
+        return self.__connection
+
+    @connection.setter
+    def connection(self, connection: MDRSConnection | None) -> None:
+        # Assignable as it always was: this is a property to check for absence on read,
+        # not to make the connection fixed once the client is built.
+        self.__connection = connection
+
+    @classmethod
+    def create_connection(
+        cls, remote: str, cache: CacheInterface | None = None, config: ConfigInterface | None = None
+    ) -> MDRSConnection:
+        if config is None:
+            config = ConfigFile(remote)
+        elif config.remote != remote:
+            raise IllegalArgumentException("Remote host parameter mismatch.")
+        if config.url is None:
+            raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
+        return MDRSConnection(config.remote, config.url, cache=cache)
+
+    def login(self, username: str, password: str) -> tuple[Token, User]:
+        user_api = UsersApi(self.connection)
+        token = user_api.token(username, password)
+        self.connection.token = token
+        user = user_api.current()
+        self.connection.user = user
+        return token, user
+
+    def logout(self) -> None:
+        self.connection.logout()
+
+    def whoami(self) -> User:
+        user_api = UsersApi(self.connection)
+        return user_api.current()
+
+    def get_laboratories(self) -> list[Laboratory]:
+        laboratory_api = LaboratoriesApi(self.connection)
+        labs = laboratory_api.list()
+        self.connection.laboratories = labs
+        return list(labs)
+
+    def find_laboratory(self, name: str) -> Laboratory:
+        if self.connection.laboratories.empty() or (self.connection.token and self.connection.token.is_expired):
+            self.get_laboratories()
+        laboratory = self.connection.laboratories.find_by_name(name)
+        if laboratory is None:
+            raise IllegalArgumentException(f"Laboratory `{name}` not found.")
+        return laboratory
+
+    def find_folder(self, laboratory: Laboratory, path: str, password: str | None = None) -> Folder:
+        folder_api = FoldersApi(self.connection)
+        folders = folder_api.list(laboratory.id, normalize("NFC", path))
+        if len(folders) != 1:
+            raise UnexpectedException(f"Folder `{path}` not found.")
+        if folders[0].lock:
+            if password is None:
+                raise UnauthorizedException(f"Folder `{path}` is locked.")
+            folder_api.auth(folders[0].id, password)
+        return folder_api.retrieve(folders[0].id)
+
+    def find_files(self, folder_id: str) -> list[File]:
+        files_api = FilesApi(self.connection)
+        page = 1
+        results_file = []
+        while page:
+            result = files_api.list(folder_id, page)
+            results_file.extend(result.results)
+            page = 0
+            if result.next:
+                page = page_num_from_url(result.next)
+        return results_file
+
+    @staticmethod
+    def is_doi(path_component: str) -> bool:
+        return path_component.startswith("10.") and "/" in path_component
+
+    @staticmethod
+    def doi_suffix_id(doi: str) -> str:
+        doi = doi.rstrip("/")
+        slash_pos = doi.find("/")
+        if slash_pos == -1:
+            return doi
+        suffix = doi[slash_pos + 1 :]
+        dot_pos = suffix.rfind(".")
+        return suffix[dot_pos + 1 :] if dot_pos != -1 else suffix
+
+    @staticmethod
+    def split_doi_and_subpath(doi_with_path: str) -> tuple[str, str]:
+        first_slash = doi_with_path.find("/")
+        if first_slash != -1:
+            after_suffix_start = first_slash + 1
+            after_first = doi_with_path[after_suffix_start:]
+            second_slash = after_first.find("/")
+            if second_slash != -1:
+                doi_end = after_suffix_start + second_slash
+                doi = doi_with_path[:doi_end]
+                subpath = doi_with_path[doi_end:]
+                if subpath == "/":
+                    return (doi, "")
+                else:
+                    return (doi, subpath)
+            else:
+                return (doi_with_path, "")
+        else:
+            return (doi_with_path, "")
+
+    @classmethod
+    def parse_remote_host(cls, path: str) -> str:
+        path_array = path.split(":")
+        remote_host = path_array[0]
+        if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2:
+            raise IllegalArgumentException("Invalid remote host")
+        return remote_host
+
+    @classmethod
+    def parse_remote_host_with_path(cls, path: str) -> tuple[str, str, str]:
+        path = re.sub(r"//+|/\./+|/\.$", "/", path)
+        if re.search(r"/\.\./|/\.\.$", path) is not None:
+            raise IllegalArgumentException("Path traversal found.")
+        path_array = path.split(":")
+        if len(path_array) != 2:
+            raise IllegalArgumentException("Invalid remote host.")
+        remote_host = path_array[0]
+        folder_array = path_array[1].split("/")
+        is_absolute_path = folder_array[0] == ""
+        if not is_absolute_path:
+            raise IllegalArgumentException("Must be absolute paths.")
+        del folder_array[0]
+        if len(folder_array) == 0:
+            laboratory = ""
+            folder = ""
+        else:
+            laboratory = folder_array.pop(0)
+            folder = "/" + "/".join(folder_array)
+        return (remote_host, laboratory, folder)
+
+    @classmethod
+    def parse_doi_remote_host(cls, path: str) -> tuple[str, str, str]:
+        parts = path.split(":", 1)
+        if len(parts) != 2:
+            raise IllegalArgumentException("remote_path must be in the form 'remote:10.xxxx/prefix.ID'")
+        remote, doi_with_path = parts
+        if not cls.is_doi(doi_with_path):
+            raise IllegalArgumentException(f"Path `{doi_with_path}` does not look like a DOI.")
+        doi, subpath = cls.split_doi_and_subpath(doi_with_path)
+        return (remote, doi, subpath)
+
+    def find_folder_by_doi(self, doi: str, password: str | None = None) -> tuple[Folder, Laboratory]:
+        doi_clean = doi.rstrip("/")
+        doi_id = self.doi_suffix_id(doi_clean)
+        doi_api = DoiApi(self.connection)
+        doi_resp = doi_api.retrieve(doi_id)
+
+        returned_doi = doi_resp.doi.rstrip("/")
+        if returned_doi.lower() != doi_clean.lower():
+            raise IllegalArgumentException(
+                f"DOI mismatch: requested `{doi_clean}` but server returned `{returned_doi}`."
+            )
+
+        folder_api = FoldersApi(self.connection)
+        folder = folder_api.retrieve(doi_resp.folder_id)
+
+        if folder.lock:
+            if password is None:
+                raise UnauthorizedException(f"Folder for DOI `{doi_clean}` is locked.")
+            folder_api.auth(doi_resp.folder_id, password)
+
+        lab_api = LaboratoriesApi(self.connection)
+        labs = lab_api.list()
+        lab = labs.find_by_id(folder.laboratory_id)
+        if lab is None:
+            raise UnexpectedException(f"Laboratory with id {folder.laboratory_id} not found.")
+
+        self.connection.laboratories = labs
+        return (folder, lab)
+
+    def resolve_folder(self, remote_path: str, password: str | None = None) -> tuple[Folder, Laboratory]:
+        path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
+        if self.is_doi(path_component):
+            remote, doi, subpath = self.parse_doi_remote_host(remote_path)
+            doi_folder, laboratory = self.find_folder_by_doi(doi, password)
+            if not subpath:
+                return (doi_folder, laboratory)
+            else:
+                abs_path = doi_folder.path.rstrip("/") + subpath
+                folder = self.find_folder(laboratory, abs_path, password)
+                return (folder, laboratory)
+        else:
+            remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
+            laboratory = self.find_laboratory(laboratory_name)
+            folder = self.find_folder(laboratory, r_path, password)
+            return (folder, laboratory)
+
+    def resolve_file(self, remote_path: str, password: str | None = None) -> tuple[Folder, Laboratory, str]:
+        path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
+        if self.is_doi(path_component):
+            remote, doi, subpath = self.parse_doi_remote_host(remote_path)
+            doi_folder, laboratory = self.find_folder_by_doi(doi, password)
+            subpath_clean = subpath.rstrip("/")
+            if not subpath_clean:
+                raise IllegalArgumentException("DOI path must point to a file, not a folder.")
+            r_dirname = os.path.dirname(subpath_clean)
+            r_basename = os.path.basename(subpath_clean)
+            abs_path = doi_folder.path.rstrip("/") + r_dirname
+            parent_folder = self.find_folder(laboratory, abs_path, password)
+            return (parent_folder, laboratory, r_basename)
+        else:
+            remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
+            r_path = r_path.rstrip("/")
+            r_dirname = os.path.dirname(r_path)
+            r_basename = os.path.basename(r_path)
+            laboratory = self.find_laboratory(laboratory_name)
+            parent_folder = self.find_folder(laboratory, r_dirname, password)
+            return (parent_folder, laboratory, r_basename)

mdrsclient/transfer.py

@@ -0,0 +1,321 @@
+import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+from unicodedata import normalize
+
+from pydantic.dataclasses import dataclass
+
+from mdrsclient.api import FilesApi, FoldersApi
+from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
+from mdrsclient.models import File, Folder, Laboratory
+from mdrsclient.models.file import find_file
+from mdrsclient.settings import CONCURRENT
+
+
+@dataclass(frozen=True)
+class UploadFileInfo:
+    folder: Folder
+    files: list[File]
+    path: str
+
+
+@dataclass(frozen=True)
+class DownloadFileInfo:
+    file: File
+    path: str
+
+
+@dataclass
+class DownloadContext:
+    isSkipIfExists: bool
+    files: list[DownloadFileInfo]
+
+
+class Uploader:
+    def __init__(self, client: Any) -> None:
+        self.client = client
+
+    def upload(
+        self, local_path: str, remote_path: str, is_recursive: bool = False, is_skip_if_exists: bool = False
+    ) -> None:
+        remote, laboratory_name, r_path = self.client.parse_remote_host_with_path(remote_path)
+        l_path = os.path.abspath(local_path)
+        if not os.path.exists(l_path):
+            raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
+
+        laboratory = self.client.find_laboratory(laboratory_name)
+        folder = self.client.find_folder(laboratory, r_path)
+        files = self.client.find_files(folder.id)
+        if os.path.isdir(l_path):
+            if not is_recursive:
+                raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
+            infos = self.__collect_directory_uploads(laboratory, r_path, l_path, folder, files)
+        else:
+            infos = [UploadFileInfo(folder, files, l_path)]
+        if not self.__multiple_upload(infos, is_skip_if_exists):
+            # One file failing is worth reporting on its own line, and worth the caller
+            # hearing about: a batch that lost files is not a batch that succeeded.
+            raise UnexpectedException("Some files failed to upload.")
+
+    def __collect_directory_uploads(
+        self, laboratory: Laboratory, r_path: str, l_path: str, folder: Folder, files: list[File]
+    ) -> list[UploadFileInfo]:
+        """Mirror a local directory tree on the remote, and list the files to send into it."""
+        infos: list[UploadFileInfo] = []
+        folder_api = FoldersApi(self.client.connection)
+        folder_map: dict[str, Folder] = {}
+        folder_map[r_path] = folder
+        files_map: dict[str, list[File]] = {}
+        files_map[r_path] = files
+        l_basename = os.path.basename(l_path)
+        for dirpath, _, filenames in os.walk(l_path, followlinks=True):
+            sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
+            d_dirname = os.path.join(r_path, sub)
+            d_basename = os.path.basename(d_dirname)
+            # prepare destination parent path
+            d_parent_dirname = os.path.dirname(d_dirname)
+            if folder_map.get(d_parent_dirname) is None:
+                parent_folder = self.client.find_folder(laboratory, d_parent_dirname)
+                folder_map[d_parent_dirname] = parent_folder
+                parent_files = self.client.find_files(parent_folder.id)
+                files_map[d_parent_dirname] = parent_files
+            # prepare destination path
+            if folder_map.get(d_dirname) is None:
+                d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
+                if d_folder is None:
+                    d_folder_id = folder_api.create(normalize("NFC", d_basename), folder_map[d_parent_dirname].id)
+                else:
+                    d_folder_id = d_folder.id
+                print(d_dirname)
+                folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
+                files_map[d_dirname] = self.client.find_files(d_folder_id)
+                if d_folder is None:
+                    folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
+            # register upload file list
+            for filename in filenames:
+                infos.append(
+                    UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
+                )
+        return infos
+
+    def __multiple_upload(self, infos: list[UploadFileInfo], is_skip_if_exists: bool) -> bool:
+        """Send every file, and report whether all of them arrived."""
+        file_api = FilesApi(self.client.connection)
+        with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
+            results = pool.map(lambda x: self.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
+            # Consumed inside the block: the results are what carry each worker's verdict.
+            return all(list(results))
+
+    def __multiple_upload_worker(self, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> bool:
+        basename = os.path.basename(info.path)
+        file = find_file(info.files, basename)
+        try:
+            if file is None:
+                file_api.create(info.folder.id, info.path)
+            elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
+                file_api.update(file, info.path)
+            print(os.path.join(info.folder.path, basename))
+        except Exception as e:
+            # Everything, not just the exceptions the API layer raises: the batch verdict
+            # is read now that the results are consumed, and a file vanishing between the
+            # walk and the upload would otherwise end the whole run with a traceback and
+            # throw away what every other file did.
+            print(f"Failed: {info.path}: {e}")
+            return False
+        return True
+
+
+class Downloader:
+    def __init__(self, client: Any) -> None:
+        self.client = client
+
+    def download(
+        self,
+        remote_path: str,
+        local_path: str,
+        is_recursive: bool = False,
+        is_skip_if_exists: bool = False,
+        password: str | None = None,
+        excludes: list[str] | None = None,
+    ) -> None:
+        if not self.__download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes):
+            # Every failure has already been printed against the file it belongs to.
+            # This is what makes the command as a whole end in failure.
+            raise UnexpectedException("Some files failed to download.")
+
+    def __download(
+        self,
+        remote_path: str,
+        local_path: str,
+        is_recursive: bool,
+        is_skip_if_exists: bool,
+        password: str | None,
+        excludes: list[str] | None,
+    ) -> bool:
+        """Fetch what the remote path names, and report whether every file arrived."""
+        excludes_clean = excludes or []
+        l_dirname = os.path.realpath(local_path)
+        if not os.path.isdir(l_dirname):
+            raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
+
+        # "remote:10.xxxx/prefix.ID[/optional/sub/path]" names a published dataset rather
+        # than a path within a laboratory, and is resolved through the DOI instead.
+        path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
+        if self.client.is_doi(path_component):
+            return self.__download_doi(
+                remote_path, l_dirname, is_recursive, is_skip_if_exists, password, excludes_clean
+            )
+
+        remote, laboratory_name, r_path = self.client.parse_remote_host_with_path(remote_path)
+        r_path = r_path.rstrip("/")
+        r_basename = os.path.basename(r_path)
+        laboratory = self.client.find_laboratory(laboratory_name)
+        r_parent_folder = self.client.find_folder(laboratory, os.path.dirname(r_path), password)
+        r_parent_files = self.client.find_files(r_parent_folder.id)
+        file = find_file(r_parent_files, r_basename)
+        if file is not None:
+            return self.__download_one(
+                excludes_clean, laboratory, r_parent_folder, file, l_dirname, r_basename, is_skip_if_exists
+            )
+        folder = r_parent_folder.find_sub_folder(r_basename)
+        if folder is None:
+            raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
+        if not is_recursive:
+            raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
+        return self.__multiple_download_pickup_recursive_files(
+            FoldersApi(self.client.connection), laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
+        )
+
+    def __download_doi(
+        self,
+        remote_path: str,
+        l_dirname: str,
+        is_recursive: bool,
+        is_skip_if_exists: bool,
+        password: str | None,
+        excludes: list[str],
+    ) -> bool:
+        """Fetch what a DOI names: the dataset's folder, or something inside it."""
+        remote, doi, subpath = self.client.parse_doi_remote_host(remote_path)
+        doi_folder, laboratory = self.client.find_folder_by_doi(doi, password)
+
+        subpath_clean = subpath.rstrip("/")
+        if not subpath_clean:
+            folder = doi_folder
+        else:
+            r_basename = os.path.basename(subpath_clean)
+            abs_path = doi_folder.path.rstrip("/") + os.path.dirname(subpath_clean)
+            r_parent_folder = self.client.find_folder(laboratory, abs_path, password)
+            file = find_file(self.client.find_files(r_parent_folder.id), r_basename)
+            if file is not None:
+                return self.__download_one(
+                    excludes, laboratory, r_parent_folder, file, l_dirname, r_basename, is_skip_if_exists
+                )
+            folder_simple = r_parent_folder.find_sub_folder(r_basename)
+            if folder_simple is None:
+                raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
+            folder = FoldersApi(self.client.connection).retrieve(folder_simple.id)
+
+        if is_recursive:
+            return self.__multiple_download_pickup_recursive_files(
+                FoldersApi(self.client.connection), laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
+            )
+        # Without -r the dataset's own files are fetched, and its sub-folders are not.
+        context = DownloadContext(is_skip_if_exists, [])
+        for file in self.client.find_files(folder.id):
+            if self.__check_excludes(excludes, laboratory, folder, file):
+                continue
+            context.files.append(DownloadFileInfo(file, os.path.join(l_dirname, file.name)))
+        return self.__multiple_download(context)
+
+    def __download_one(
+        self,
+        excludes: list[str],
+        laboratory: Laboratory,
+        folder: Folder,
+        file: File,
+        l_dirname: str,
+        local_name: str,
+        is_skip_if_exists: bool,
+    ) -> bool:
+        """
+        Fetch a single named file into the local directory.
+
+        Saved under the name the caller asked for rather than the one the server holds:
+        the two are matched case-insensitively, so they need not be spelled alike.
+        """
+        if self.__check_excludes(excludes, laboratory, folder, file):
+            return True
+        context = DownloadContext(is_skip_if_exists, [])
+        context.files.append(DownloadFileInfo(file, os.path.join(l_dirname, local_name)))
+        return self.__multiple_download(context)
+
+    def __multiple_download_pickup_recursive_files(
+        self,
+        folder_api: FoldersApi,
+        laboratory: Laboratory,
+        folder_id: str,
+        basedir: str,
+        excludes: list[str],
+        is_skip_if_exists: bool,
+    ) -> bool:
+        context = DownloadContext(is_skip_if_exists, [])
+        try:
+            folder = folder_api.retrieve(folder_id)
+            files = self.client.find_files(folder.id)
+        except Exception as e:
+            print(f"Failed: {basedir}: {e}")
+            return False
+        dirname = os.path.join(basedir, folder.name)
+        if self.__check_excludes(excludes, laboratory, folder, None):
+            return True
+        try:
+            # `exist_ok` rather than a prior check: two workers can reach the same parent.
+            os.makedirs(dirname, exist_ok=True)
+        except OSError as e:
+            # One folder the client cannot make locally is not a reason to abandon its
+            # siblings, which is what this walk now promises.
+            print(f"Failed: {dirname}: {e}")
+            return False
+        print(dirname)
+        for file in files:
+            if self.__check_excludes(excludes, laboratory, folder, file):
+                continue
+            path = os.path.join(dirname, file.name)
+            context.files.append(DownloadFileInfo(file, path))
+        succeeded = self.__multiple_download(context)
+        # A folder that lost a file is still a folder whose sub-folders the user asked
+        # for, so the walk carries on and the verdict is collected for the caller.
+        for sub_folder in folder.sub_folders:
+            if not self.__multiple_download_pickup_recursive_files(
+                folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
+            ):
+                succeeded = False
+        return succeeded
+
+    def __multiple_download(self, context: DownloadContext) -> bool:
+        """Fetch every file in the batch, and report whether all of them arrived."""
+        file_api = FilesApi(self.client.connection)
+        with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
+            results = pool.map(
+                lambda x: self.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
+            )
+            # Consumed inside the block, and in full: every worker's verdict counts, not
+            # just the first refusal.
+            return all(list(results))
+
+    def __multiple_download_worker(self, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
+        if not is_skip_if_exists or not os.path.exists(info.path) or info.file.size != os.path.getsize(info.path):
+            try:
+                file_api.download(info.file, info.path)
+            except Exception as e:
+                # Nothing to clear up: a failed transfer writes only to its own scratch
+                # file beside the destination, and removes that itself.
+                print(f"Failed: {info.path}: {e}")
+                return False
+        print(info.path)
+        return True
+
+    def __check_excludes(self, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
+        path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
+        return path in excludes

mdrsclient/utils.py

@@ -1,5 +1,7 @@
 import os
-from typing import IO, Any
+import time
+from typing import IO, Any, Final
+from urllib.parse import parse_qs, urlparse
 
 if os.name == "nt":
     import msvcrt
@@ -8,10 +10,23 @@ elif os.name == "posix":
 
 
 class FileLock:
+    # Long enough to outlast a token refresh, which is what the lock is held across.
+    WAIT_SECONDS: Final[float] = 60.0
+
     @staticmethod
     def lock(file: IO[Any]) -> None:
         if os.name == "nt":
-            msvcrt.locking(file.fileno(), msvcrt.LK_LOCK, 1)
+            # msvcrt.LK_LOCK gives up after ten one-second attempts, which is shorter
+            # than the refresh it now has to wait for, so do the waiting here instead.
+            deadline = time.monotonic() + FileLock.WAIT_SECONDS
+            while True:
+                try:
+                    msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
+                    return
+                except OSError:
+                    if time.monotonic() >= deadline:
+                        raise
+                    time.sleep(0.1)
         elif os.name == "posix":
             fcntl.flock(file.fileno(), fcntl.LOCK_EX)
 
@@ -21,3 +36,10 @@ class FileLock:
             msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
         elif os.name == "posix":
             fcntl.flock(file.fileno(), fcntl.LOCK_UN)
+
+
+def page_num_from_url(url: str) -> int | None:
+    parsed_url = urlparse(url)
+    params = parse_qs(parsed_url.query)
+    page = params.get("page", [None])[0]
+    return int(page) if page is not None else None

pyproject.toml

@@ -0,0 +1,91 @@
+[build-system]
+requires = ["flit_core>=4.0.2,<5.0.0"]
+build-backend = "flit_core.buildapi"
+
+[tool.flit.module]
+name = "mdrsclient"
+
+[project]
+name = "mdrs-client-python"
+version = "1.3.19"
+description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
+authors = [
+    { name = "Yoshihiro OKUMURA", email = "yoshihiro.okumura@riken.jp" }
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "OSI Approved :: MIT License",
+    "Topic :: Utilities",
+]
+
+dependencies = [
+    "requests>=2.34.2",
+    "requests-toolbelt>=1.0.0",
+    "python-dotenv>=1.2.3",
+    "pydantic>=2.13.5",
+    "pydantic-settings>=2.15.0",
+    "PyJWT>=2.13.0",
+    "validators>=0.35.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "black>=26.5.1",
+    "flake8>=7.3.0",
+    "Flake8-pyproject>=1.2.4",
+    "isort>=9.0.1",
+    "pyright>=1.1.411",
+]
+
+[project.scripts]
+mdrs = "mdrsclient.__main__:main"
+
+[tool.black]
+line-length = 120
+target-version = ['py310']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.venv
+  | __pycache__
+  | dist
+)/
+'''
+
+[tool.flake8]
+exclude = ".git, .venv, __pycache__, dist"
+max-complexity = 10
+max-line-length = 120
+# E203 flags the space black puts before a slice colon. Black formats this project, so
+# its output is the standard and flake8 has to accept it.
+extend-ignore = ["E203"]
+
+[tool.isort]
+profile = "black"
+line_length = 120
+
+[tool.mypy]
+
+[tool.pyright]
+typeCheckingMode = "basic"
+# This project keeps its environment in `./.venv`. Named explicitly because without it
+# pyright falls back to the system interpreter, cannot resolve pydantic, and reports every
+# model field as an unknown argument. An environment kept elsewhere gets that same fallback
+# and can override it with a local `pyrightconfig.json` or `pyright --pythonpath`.
+venvPath = "."
+venv = ".venv"
+exclude = ["**/__pycache__", "**/.*", "dist"]
+#reportUnknownMemberType = "warning"
+#reportUnknownVariableType = "warning"

requirements.txt

@@ -1,6 +0,0 @@
-requests
-python-dotenv
-pydantic
-pydantic-settings
-PyJWT
-validators

setup.py

@@ -1,40 +0,0 @@
-import os
-from typing import Final
-
-from setuptools import find_packages, setup
-
-from mdrsclient import __version__
-
-BASE_DIR: Final[str] = os.path.realpath(os.path.dirname(__file__))
-
-with open(os.path.join(BASE_DIR, "requirements.txt")) as f:
-    __requirements__ = f.read().splitlines()
-
-with open(os.path.join(BASE_DIR, "README.md")) as f:
-    __readme__ = f.read()
-
-setup(
-    name="mdrsclient",
-    version=__version__,
-    description="A MDRS command-line tool",
-    long_description=__readme__,
-    author="Neuroinformatics Unit, RIKEN CBS",
-    license="MIT",
-    classifiers=[
-        "Development Status :: 3 - Alpha",
-        "Environment :: Console",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Science/Research",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "OSI Approved :: MIT License",
-        "Topic :: Utilities",
-    ],
-    packages=find_packages(),
-    include_package_data=True,
-    package_data={
-        "mdrsclient": ["VERSION"],
-    },
-    install_requires=__requirements__,
-    entry_points={"console_scripts": ["mdrs=mdrsclient.__main__:main"]},
-)

tests/__init__.py

@@ -0,0 +1 @@
+# Mark tests directory as a Python package

tests/test_client.py

@@ -0,0 +1,165 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from mdrsclient.client import MdrsClient
+from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models import File, Folder, FolderSimple, Laboratories, Laboratory
+
+TIMESTAMP = "2026-01-01T00:00:00+09:00"
+LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory")
+
+
+def make_file(id: str, name: str) -> File:
+    return File(
+        id=id,
+        name=name,
+        type="text/plain",
+        size=1,
+        thumbnail=None,
+        description="",
+        metadata={},
+        download_url=f"v3/files/{id}/download/",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+    )
+
+
+def make_sub_folder(id: str, name: str) -> FolderSimple:
+    return FolderSimple(
+        id=id,
+        pid=None,
+        name=name,
+        access_level=1,
+        lock=False,
+        size=0,
+        laboratory_id=1,
+        description="",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+        restrict_opened_at=None,
+    )
+
+
+def make_folder(id: str, name: str, path: str, sub_folders: list[FolderSimple] | None = None) -> Folder:
+    return Folder(
+        id=id,
+        pid=None,
+        name=name,
+        access_level=1,
+        lock=False,
+        size=0,
+        laboratory_id=1,
+        description="",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+        restrict_opened_at=None,
+        metadata=[],
+        sub_folders=sub_folders if sub_folders is not None else [],
+        path=path,
+    )
+
+
+class TestLsColumnWidths(unittest.TestCase):
+    """Each column is as wide as the widest thing printed under it."""
+
+    def test_the_size_column_fits_a_sub_folder_larger_than_its_parent(self):
+        from unittest.mock import MagicMock as _MagicMock
+
+        from mdrsclient.commands.ls import LsCommand, LsCommandContext
+
+        laboratories = Laboratories()
+        laboratories.append(LABORATORY)
+        connection = _MagicMock()
+        connection.laboratories = laboratories
+        client = MdrsClient(connection)
+
+        big = make_sub_folder("s1", "big")
+        object.__setattr__(big, "size", 123456789)
+        parent = make_folder("f1", "root", "/root/", [big])
+        context = LsCommandContext("remote:/mylab", client, LABORATORY, "", False, False, False)
+
+        widths = LsCommand._column_widths(context, parent, [])
+
+        self.assertEqual(widths["size"], len("123456789"))
+
+
+class TestCopyAndMove(unittest.TestCase):
+    """`cp` and `mv` resolve both ends the same way and differ only in what they call."""
+
+    def make_client(self, folders_by_path: dict, files_by_folder: dict) -> MdrsClient:
+        client = MdrsClient(MagicMock())
+        client.find_laboratory = MagicMock(return_value=LABORATORY)
+        client.find_folder = MagicMock(side_effect=lambda lab, path, password=None: folders_by_path[path])
+        client.find_files = MagicMock(side_effect=lambda folder_id: files_by_folder.get(folder_id, []))
+        return client
+
+    def make_tree(self, source_has: str) -> tuple[MdrsClient, dict, dict]:
+        sub = make_sub_folder("fsub", "sub")
+        folder_a = make_folder("fa", "a", "/a/", [sub] if source_has == "folder" else [])
+        folder_b = make_folder("fb", "b", "/b/")
+        folders = {"/a": folder_a, "/b": folder_b, "/b/": folder_b}
+        files = {"fa": [make_file("x1", "data.txt")] if source_has == "file" else [], "fb": []}
+        return self.make_client(folders, files), folders, files
+
+    def test_a_file_is_copied_into_the_destination_folder(self):
+        client, _, files = self.make_tree("file")
+        with patch("mdrsclient.client.FilesApi") as files_api:
+            client.cp("myremote:/mylab/a/data.txt", "myremote:/mylab/b/data.txt")
+            files_api.return_value.copy.assert_called_once_with(files["fa"][0], "fb", "data.txt")
+
+    def test_a_file_is_moved_into_the_destination_folder(self):
+        client, _, files = self.make_tree("file")
+        with patch("mdrsclient.client.FilesApi") as files_api:
+            client.mv("myremote:/mylab/a/data.txt", "myremote:/mylab/b/data.txt")
+            files_api.return_value.move.assert_called_once_with(files["fa"][0], "fb", "data.txt")
+
+    def test_a_trailing_separator_keeps_the_source_name(self):
+        client, _, files = self.make_tree("file")
+        with patch("mdrsclient.client.FilesApi") as files_api:
+            client.mv("myremote:/mylab/a/data.txt", "myremote:/mylab/b/")
+            files_api.return_value.move.assert_called_once_with(files["fa"][0], "fb", "data.txt")
+
+    def test_copying_a_folder_needs_the_recursive_flag(self):
+        client, _, _ = self.make_tree("folder")
+        with patch("mdrsclient.client.FoldersApi") as folders_api:
+            with self.assertRaises(IllegalArgumentException) as caught:
+                client.cp("myremote:/mylab/a/sub", "myremote:/mylab/b/sub")
+            self.assertIn("Is a folder", str(caught.exception))
+            folders_api.return_value.copy.assert_not_called()
+
+    def test_a_folder_is_copied_when_recursive(self):
+        client, folders, _ = self.make_tree("folder")
+        with patch("mdrsclient.client.FoldersApi") as folders_api:
+            client.cp("myremote:/mylab/a/sub", "myremote:/mylab/b/sub", is_recursive=True)
+            folders_api.return_value.copy.assert_called_once_with(folders["/a"].sub_folders[0], "fb", "sub")
+
+    def test_a_folder_is_moved_without_the_recursive_flag(self):
+        client, folders, _ = self.make_tree("folder")
+        with patch("mdrsclient.client.FoldersApi") as folders_api:
+            client.mv("myremote:/mylab/a/sub", "myremote:/mylab/b/sub")
+            folders_api.return_value.move.assert_called_once_with(folders["/a"].sub_folders[0], "fb", "sub")
+
+    def test_a_destination_that_already_holds_the_name_is_refused(self):
+        sub = make_sub_folder("fsub", "sub")
+        folders = {"/a": make_folder("fa", "a", "/a/", [sub]), "/b": make_folder("fb", "b", "/b/")}
+        files = {"fa": [], "fb": [make_file("x2", "sub")]}
+        client = self.make_client(folders, files)
+        with self.assertRaises(IllegalArgumentException) as caught:
+            client.mv("myremote:/mylab/a/sub", "myremote:/mylab/b/sub")
+        self.assertIn("Cannot overwrite non-folder", str(caught.exception))
+
+    def test_a_transfer_across_laboratories_is_refused(self):
+        client, _, _ = self.make_tree("file")
+        with self.assertRaises(IllegalArgumentException) as caught:
+            client.cp("myremote:/mylab/a/data.txt", "myremote:/otherlab/b/data.txt")
+        self.assertIn("Laboratory mismatched", str(caught.exception))
+
+    def test_a_source_that_does_not_exist_is_refused(self):
+        client, _, _ = self.make_tree("file")
+        with self.assertRaises(IllegalArgumentException) as caught:
+            client.mv("myremote:/mylab/a/missing.txt", "myremote:/mylab/b/missing.txt")
+        self.assertIn("not found", str(caught.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_commands.py

@@ -0,0 +1,418 @@
+import argparse
+import json
+import unittest
+from io import StringIO
+from unittest.mock import MagicMock, patch
+
+from mdrsclient.client import MdrsClient
+from mdrsclient.commands import (
+    ChaclCommand,
+    ConfigCommand,
+    CpCommand,
+    DownloadCommand,
+    FileMetadataCommand,
+    LabsCommand,
+    LoginCommand,
+    LogoutCommand,
+    LsCommand,
+    MetadataCommand,
+    MkdirCommand,
+    MvCommand,
+    RmCommand,
+    UploadCommand,
+    VersionCommand,
+    WhoamiCommand,
+)
+from mdrsclient.models import Folder, Laboratory
+
+
+class TestCommands(unittest.TestCase):
+    def parse_args(self, cmd_class, args_list):
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers(title="subcommands")
+        cmd_class.register(subparsers)
+        return parser.parse_args(args_list)
+
+    def test_version_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.return_value = mock_client
+            mock_client.version.return_value = "mdrs 1.3.17"
+
+            args = self.parse_args(VersionCommand, ["version"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertEqual(fake_out.getvalue().strip(), "mdrs 1.3.17")
+
+            mock_client_class.assert_called_once_with(None)
+            mock_client.version.assert_called_once()
+
+    def test_login_command_with_args(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+
+            args = self.parse_args(LoginCommand, ["login", "-u", "myuser", "-p", "mypass", "myremote"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertIn("Login Successful", fake_out.getvalue())
+
+            mock_client_class.parse_remote_host.assert_called_once_with("myremote")
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.login.assert_called_once_with("myuser", "mypass")
+
+    def test_login_command_interactive(self):
+        with (
+            patch("mdrsclient.client.MdrsClient") as mock_client_class,
+            patch("builtins.input", return_value="myuser_int") as mock_input,
+            patch("getpass.getpass", return_value="mypass_int") as mock_getpass,
+        ):
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+
+            args = self.parse_args(LoginCommand, ["login", "myremote"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertIn("Login Successful", fake_out.getvalue())
+
+            mock_input.assert_called_once_with("Username: ")
+            mock_getpass.assert_called_once_with("Password: ")
+            mock_client.login.assert_called_once_with("myuser_int", "mypass_int")
+
+    def test_logout_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+
+            args = self.parse_args(LogoutCommand, ["logout", "myremote"])
+            args.func(args)
+
+            mock_client_class.parse_remote_host.assert_called_once_with("myremote")
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.logout.assert_called_once()
+
+    def test_whoami_command_logged_in(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+            mock_client.connection.token = None
+
+            mock_user = MagicMock()
+            mock_user.username = "test_user"
+            mock_client.whoami.return_value = mock_user
+
+            args = self.parse_args(WhoamiCommand, ["whoami", "myremote"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertEqual(fake_out.getvalue().strip(), "test_user")
+
+            mock_client_class.parse_remote_host.assert_called_once_with("myremote")
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.whoami.assert_called_once()
+
+    def test_whoami_command_anonymous(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+            mock_client.connection.token = None
+            mock_client.whoami.side_effect = Exception("Not logged in")
+
+            args = self.parse_args(WhoamiCommand, ["whoami", "myremote"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertEqual(fake_out.getvalue().strip(), "(Anonymous)")
+
+    def test_labs_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client_class.parse_remote_host.return_value = "myremote"
+
+            mock_lab = MagicMock()
+            mock_lab.id = 1
+            mock_lab.name = "lab_name"
+            mock_lab.pi_name = "pi_name"
+            mock_lab.full_name = "full_name"
+            mock_client.get_laboratories.return_value = [mock_lab]
+
+            args = self.parse_args(LabsCommand, ["labs", "myremote"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                output = fake_out.getvalue()
+                self.assertIn("Name", output)
+                self.assertIn("lab_name", output)
+                self.assertIn("pi_name", output)
+
+            mock_client_class.parse_remote_host.assert_called_once_with("myremote")
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.get_laboratories.assert_called_once()
+
+    def test_ls_command_plain(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock(spec=MdrsClient)
+            mock_client_class.from_remote.return_value = mock_client
+
+            mock_folder = Folder(
+                id="folder_id",
+                pid=None,
+                name="root",
+                access_level=1,
+                lock=False,
+                size=0,
+                laboratory_id=1,
+                description="",
+                created_at="2026-07-02T00:00:00Z",
+                updated_at="2026-07-02T00:00:00Z",
+                restrict_opened_at=None,
+                metadata=[],
+                sub_folders=[],
+                path="/root",
+            )
+
+            mock_lab = Laboratory(id=1, name="mylab", pi_name="pi_name", full_name="full_name")
+
+            mock_client.resolve_folder.return_value = (mock_folder, mock_lab)
+
+            mock_file = MagicMock()
+            mock_file.name = "file.txt"
+            mock_file.size = 100
+            mock_file.updated_at_name = "2026-07-02"
+            mock_client.find_files.return_value = [mock_file]
+
+            args = self.parse_args(LsCommand, ["ls", "myremote:/mylab/"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                output = fake_out.getvalue()
+                self.assertIn("Type", output)
+                self.assertIn("file.txt", output)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.resolve_folder.assert_called_once_with("myremote:/mylab/", None)
+            mock_client.find_files.assert_called_once_with(mock_folder.id)
+
+    def test_ls_command_json(self):
+        with (
+            patch("mdrsclient.client.MdrsClient") as mock_client_class,
+            patch("mdrsclient.commands.ls.FoldersApi") as mock_folders_api_class,
+        ):
+            mock_folders_api = MagicMock()
+            mock_folders_api_class.return_value = mock_folders_api
+            mock_folders_api.metadata.return_value = {"folder_meta": "val"}
+
+            mock_client = MagicMock(spec=MdrsClient)
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client.connection = MagicMock()
+
+            mock_folder = Folder(
+                id="folder_id",
+                pid="parent_id",
+                name="root",
+                access_level=1,
+                lock=False,
+                size=0,
+                laboratory_id=1,
+                description="Root folder",
+                created_at="2026-07-02T00:00:00Z",
+                updated_at="2026-07-02T00:00:00Z",
+                restrict_opened_at=None,
+                metadata=[],
+                sub_folders=[],
+                path="/root",
+            )
+
+            mock_lab = Laboratory(id=1, name="mylab", pi_name="pi_name", full_name="full_name")
+
+            mock_client.resolve_folder.return_value = (mock_folder, mock_lab)
+            mock_client.connection.laboratories.find_by_id.return_value = mock_lab
+
+            mock_file = MagicMock()
+            mock_file.id = "file_id"
+            mock_file.name = "file.txt"
+            mock_file.type = "text"
+            mock_file.size = 100
+            mock_file.description = "A file"
+            mock_file.metadata = {}
+            mock_file.download_url = "download/file"
+            mock_file.created_at = "2026-07-02T00:00:00Z"
+            mock_file.updated_at = "2026-07-02T00:00:00Z"
+            mock_client.find_files.return_value = [mock_file]
+
+            args = self.parse_args(LsCommand, ["ls", "-J", "myremote:/mylab/"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                output = fake_out.getvalue()
+                parsed_json = json.loads(output)
+                self.assertEqual(parsed_json["name"], "root")
+                self.assertEqual(parsed_json["files"][0]["name"], "file.txt")
+
+    def test_mkdir_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(MkdirCommand, ["mkdir", "myremote:/mylab/newfolder"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.mkdir.assert_called_once_with("myremote:/mylab/newfolder")
+
+    def test_rm_command_file(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(RmCommand, ["rm", "myremote:/mylab/file.txt"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.rm.assert_called_once_with("myremote:/mylab/file.txt", False)
+
+    def test_rm_command_recursive(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(RmCommand, ["rm", "-r", "myremote:/mylab/folder"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.rm.assert_called_once_with("myremote:/mylab/folder", True)
+
+    def test_cp_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(CpCommand, ["cp", "myremote:/mylab/src", "myremote:/mylab/dest"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.cp.assert_called_once_with("myremote:/mylab/src", "myremote:/mylab/dest", False)
+
+    def test_mv_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(MvCommand, ["mv", "myremote:/mylab/src", "myremote:/mylab/dest"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.mv.assert_called_once_with("myremote:/mylab/src", "myremote:/mylab/dest")
+
+    def test_chacl_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(ChaclCommand, ["chacl", "-r", "-p", "secret", "private", "myremote:/mylab/"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.chacl.assert_called_once_with("myremote:/mylab/", 1, True, "secret")
+
+    def test_metadata_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client.metadata.return_value = {"meta_key": "meta_val"}
+
+            args = self.parse_args(MetadataCommand, ["metadata", "-p", "secret", "myremote:/mylab/"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                parsed_json = json.loads(fake_out.getvalue())
+                self.assertEqual(parsed_json["meta_key"], "meta_val")
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.metadata.assert_called_once_with("myremote:/mylab/", "secret")
+
+    def test_file_metadata_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+            mock_client.file_metadata.return_value = {"file_meta": "val"}
+
+            args = self.parse_args(FileMetadataCommand, ["file-metadata", "-p", "secret", "myremote:/mylab/file.txt"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                parsed_json = json.loads(fake_out.getvalue())
+                self.assertEqual(parsed_json["file_meta"], "val")
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.file_metadata.assert_called_once_with("myremote:/mylab/file.txt", "secret")
+
+    def test_upload_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(UploadCommand, ["upload", "-r", "-s", "local_file.txt", "myremote:/mylab/"])
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.upload.assert_called_once_with("local_file.txt", "myremote:/mylab/", True, True)
+
+    def test_download_command(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.from_remote.return_value = mock_client
+
+            args = self.parse_args(
+                DownloadCommand, ["download", "-r", "-s", "-p", "pass", "-e", "ex1", "myremote:/mylab/", "local_dir"]
+            )
+            args.func(args)
+
+            mock_client_class.from_remote.assert_called_once_with("myremote")
+            mock_client.download.assert_called_once_with("myremote:/mylab/", "local_dir", True, True, "pass", ["ex1"])
+
+    def test_config_create(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.return_value = mock_client
+
+            args = self.parse_args(ConfigCommand, ["config", "create", "myremote", "http://example.com"])
+            args.func(args)
+
+            mock_client_class.assert_called_once_with(None)
+            mock_client.config_create.assert_called_once_with("myremote", "http://example.com")
+
+    def test_config_update(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.return_value = mock_client
+
+            args = self.parse_args(ConfigCommand, ["config", "update", "myremote", "http://example.com"])
+            args.func(args)
+
+            mock_client_class.assert_called_once_with(None)
+            mock_client.config_update.assert_called_once_with("myremote", "http://example.com")
+
+    def test_config_delete(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.return_value = mock_client
+
+            args = self.parse_args(ConfigCommand, ["config", "delete", "myremote"])
+            args.func(args)
+
+            mock_client_class.assert_called_once_with(None)
+            mock_client.config_delete.assert_called_once_with("myremote")
+
+    def test_config_list(self):
+        with patch("mdrsclient.client.MdrsClient") as mock_client_class:
+            mock_client = MagicMock()
+            mock_client_class.return_value = mock_client
+            mock_client.config_list.return_value = [("remote1", "url1"), ("remote2", "url2")]
+
+            args = self.parse_args(ConfigCommand, ["config", "list"])
+            with patch("sys.stdout", new=StringIO()) as fake_out:
+                args.func(args)
+                self.assertEqual(fake_out.getvalue(), "remote1:\turl1\nremote2:\turl2\n")
+
+            mock_client_class.assert_called_once_with(None)
+            mock_client.config_list.assert_called_once()

tests/test_services.py

@@ -0,0 +1,55 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from mdrsclient.models import Doi, Folder, Laboratories, Laboratory
+from mdrsclient.services import MdrsService
+
+TIMESTAMP = "2026-01-01T00:00:00+09:00"
+DOI = "10.60178/cbs.20260429-001"
+LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory")
+
+
+def make_locked_folder() -> Folder:
+    return Folder(
+        id="f1",
+        pid=None,
+        name="root",
+        access_level=4,
+        lock=True,
+        size=0,
+        laboratory_id=1,
+        description="",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+        restrict_opened_at=None,
+        metadata=[],
+        sub_folders=[],
+        path="/root/",
+    )
+
+
+class TestFindFolderByDoi(unittest.TestCase):
+    """The DOI response carries the folder id as a field, not as a nested object."""
+
+    def test_a_locked_doi_folder_is_unlocked_with_its_folder_id(self):
+        laboratories = Laboratories()
+        laboratories.append(LABORATORY)
+
+        with (
+            patch("mdrsclient.services.DoiApi") as doi_api,
+            patch("mdrsclient.services.FoldersApi") as folders_api,
+            patch("mdrsclient.services.LaboratoriesApi") as laboratories_api,
+        ):
+            doi_api.return_value.retrieve.return_value = Doi(id="20260429-001", doi=DOI, folder_id="f1")
+            folders_api.return_value.retrieve.return_value = make_locked_folder()
+            laboratories_api.return_value.list.return_value = laboratories
+
+            folder, laboratory = MdrsService(MagicMock()).find_folder_by_doi(DOI, "secret")
+
+            folders_api.return_value.auth.assert_called_once_with("f1", "secret")
+            self.assertEqual(folder.id, "f1")
+            self.assertEqual(laboratory.name, "mylab")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_token_cache.py

@@ -0,0 +1,198 @@
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+import jwt
+
+from mdrsclient.api.utils import retry_on_expired_token, token_check, token_recover
+from mdrsclient.cache import CacheFile
+from mdrsclient.config import normalize_url
+from mdrsclient.connection import MDRSConnection
+from mdrsclient.exceptions import IllegalArgumentException, ServerBusyException, TokenExpiredException
+from mdrsclient.models import Token
+
+REMOTE = "unittest"
+
+
+def make_token(access_offset: int, refresh_offset: int, label: str) -> Token:
+    now = int(time.time())
+
+    def encode(token_type: str, offset: int) -> str:
+        return jwt.encode(
+            {
+                "token_type": token_type,
+                "exp": now + offset,
+                "iat": now,
+                "jti": f"{label}-{token_type}",
+                "user_id": 1,
+            },
+            "unittest-signing-key-not-verified-anywhere",
+        )
+
+    return Token(access=encode("access", access_offset), refresh=encode("refresh", refresh_offset))
+
+
+class TestTokenCache(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        patcher = patch("mdrsclient.cache.CONFIG_DIRNAME", self.tmp.name)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        self.addCleanup(self.tmp.cleanup)
+
+    def test_reload_picks_up_a_write_from_another_holder(self) -> None:
+        """The cache is shared, so a value read earlier can already be out of date."""
+        reader = CacheFile(REMOTE)
+        writer = CacheFile(REMOTE)
+
+        first = make_token(3600, 86400, "first")
+        writer.token = first
+        self.assertEqual(reader.token, first)
+
+        second = make_token(3600, 86400, "second")
+        writer.token = second
+
+        reader.reload()
+        self.assertEqual(reader.token, second)
+
+    def test_token_check_uses_the_token_another_holder_just_wrote(self) -> None:
+        """A rotating provider drops the token it replaces, so the refresh must not
+        be sent again once someone else has already made the round trip."""
+        connection = MDRSConnection(REMOTE, "http://localhost:8000/api/")
+        connection.token = make_token(-60, 86400, "stale")
+
+        # Another process refreshes while this one is between requests.
+        other = CacheFile(REMOTE)
+        rotated = make_token(3600, 86400, "rotated")
+        other.token = rotated
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            token_check(connection)
+
+        users_api.assert_not_called()
+        self.assertEqual(connection.token, rotated)
+
+    def test_token_check_refreshes_when_nothing_else_has(self) -> None:
+        connection = MDRSConnection(REMOTE, "http://localhost:8000/api/")
+        connection.token = make_token(-60, 86400, "stale")
+        rotated = make_token(3600, 86400, "rotated")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = rotated
+            token_check(connection)
+
+        users_api.return_value.tokenRefresh.assert_called_once()
+        self.assertEqual(connection.token, rotated)
+
+    def test_token_check_logs_out_once_the_refresh_token_expires(self) -> None:
+        connection = MDRSConnection(REMOTE, "http://localhost:8000/api/")
+        connection.token = make_token(-3600, -60, "dead")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            token_check(connection)
+
+        users_api.assert_not_called()
+        self.assertIsNone(connection.token)
+
+
+class TestUrlNormalization(unittest.TestCase):
+    """Both clients share config.ini, so they have to agree on what a remote URL is."""
+
+    def test_trailing_slash_is_dropped(self) -> None:
+        self.assertEqual(normalize_url("http://127.0.0.1:8000/api/"), "http://127.0.0.1:8000/api")
+        self.assertEqual(normalize_url("https://neurodata.riken.jp/api/"), "https://neurodata.riken.jp/api")
+
+    def test_bare_hostname_is_accepted(self) -> None:
+        self.assertEqual(normalize_url("http://localhost:8000/api"), "http://localhost:8000/api")
+
+    def test_only_http_schemes_are_accepted(self) -> None:
+        for url in ("ftp://x.example.com/", "file:///etc/passwd", "not-a-url", "http://"):
+            with self.subTest(url=url), self.assertRaises(IllegalArgumentException):
+                normalize_url(url)
+
+
+class TestExpiredTokenRetry(unittest.TestCase):
+    """A request can wait in the server's queue for longer than its access token lives."""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        patcher = patch("mdrsclient.cache.CONFIG_DIRNAME", self.tmp.name)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        self.addCleanup(self.tmp.cleanup)
+        self.connection = MDRSConnection(REMOTE, "http://localhost:8000/api")
+        self.connection.token = make_token(3600, 86400, "sent")
+
+    def test_a_token_someone_else_refreshed_is_reused_rather_than_replaced(self) -> None:
+        used = self.connection.token
+        rotated = make_token(3600, 86400, "rotated")
+        CacheFile(REMOTE).token = rotated  # another process, while the request waited
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            recovered = token_recover(self.connection, used)
+
+        self.assertTrue(recovered)
+        users_api.assert_not_called()
+        self.assertEqual(self.connection.token, rotated)
+        # The retry carries whatever the session header says, not what the cache holds,
+        # so reading the new token through is not enough on its own.
+        self.assertEqual(self.connection.session.headers["Authorization"], f"Bearer {rotated.access}")
+
+    def test_the_token_is_refreshed_when_nobody_else_has(self) -> None:
+        used = self.connection.token
+        rotated = make_token(3600, 86400, "rotated")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = rotated
+            recovered = token_recover(self.connection, used)
+
+        self.assertTrue(recovered)
+        users_api.return_value.tokenRefresh.assert_called_once()
+        self.assertEqual(self.connection.token, rotated)
+
+    def test_the_request_is_sent_once_more_and_only_once(self) -> None:
+        calls: list[str] = []
+
+        class Api:
+            def __init__(self, connection: MDRSConnection) -> None:
+                self.connection = connection
+
+            @retry_on_expired_token
+            def send(self) -> str:
+                calls.append("sent")
+                if len(calls) == 1:
+                    raise TokenExpiredException("expired on arrival")
+                return "ok"
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = make_token(3600, 86400, "rotated")
+            self.assertEqual(Api(self.connection).send(), "ok")
+
+        self.assertEqual(len(calls), 2)
+
+    def test_a_second_refusal_is_reported_as_an_overloaded_server(self) -> None:
+        calls: list[str] = []
+
+        class Api:
+            def __init__(self, connection: MDRSConnection) -> None:
+                self.connection = connection
+
+            @retry_on_expired_token
+            def send(self) -> str:
+                calls.append("sent")
+                raise TokenExpiredException("expired on arrival")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = make_token(3600, 86400, "rotated")
+            with self.assertRaises(ServerBusyException) as caught:
+                Api(self.connection).send()
+
+        # Sent twice and no more, and the message names the cause the user can act on
+        # rather than an authentication failure that would send them to the login form.
+        self.assertEqual(len(calls), 2)
+        self.assertIn("overloaded", str(caught.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_transfer.py

@@ -0,0 +1,297 @@
+import os
+import tempfile
+import unittest
+from io import StringIO
+from unittest.mock import MagicMock, patch
+
+from mdrsclient.api import FilesApi
+from mdrsclient.exceptions import MDRSException, UnexpectedException
+from mdrsclient.models import File, Folder, FolderSimple, Laboratory
+from mdrsclient.transfer import Downloader, Uploader
+
+TIMESTAMP = "2026-01-01T00:00:00+09:00"
+
+
+def make_file(id: str, name: str, size: int = 1) -> File:
+    return File(
+        id=id,
+        name=name,
+        type="text/plain",
+        size=size,
+        thumbnail=None,
+        description="",
+        metadata={},
+        download_url=f"v3/files/{id}/download/",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+    )
+
+
+def make_folder_simple(id: str, name: str) -> FolderSimple:
+    return FolderSimple(
+        id=id,
+        pid=None,
+        name=name,
+        access_level=1,
+        lock=False,
+        size=0,
+        laboratory_id=1,
+        description="",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+        restrict_opened_at=None,
+    )
+
+
+def make_folder(id: str, name: str, path: str, sub_folders: list[FolderSimple] | None = None) -> Folder:
+    return Folder(
+        id=id,
+        pid=None,
+        name=name,
+        access_level=1,
+        lock=False,
+        size=0,
+        laboratory_id=1,
+        description="",
+        created_at=TIMESTAMP,
+        updated_at=TIMESTAMP,
+        restrict_opened_at=None,
+        metadata=[],
+        sub_folders=sub_folders if sub_folders is not None else [],
+        path=path,
+    )
+
+
+LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory")
+
+
+class TestUploadReportsFailure(unittest.TestCase):
+    """A file that never reached the server must not leave the command reporting success."""
+
+    def make_client(self, folder: Folder, files: list[File]) -> MagicMock:
+        client = MagicMock()
+        client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/")
+        client.find_laboratory.return_value = LABORATORY
+        client.find_folder.return_value = folder
+        client.find_files.return_value = files
+        return client
+
+    def test_a_refused_upload_is_raised_to_the_caller(self):
+        client = self.make_client(make_folder("f1", "root", "/"), [])
+        with tempfile.TemporaryDirectory() as tmp:
+            local = os.path.join(tmp, "data.txt")
+            with open(local, "w") as f:
+                f.write("x")
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
+                with self.assertRaises(UnexpectedException):
+                    Uploader(client).upload(local, "myremote:/mylab/", False, False)
+
+    def test_an_error_the_api_layer_did_not_raise_is_still_counted(self):
+        """A file that vanished between the walk and the upload must not end the run."""
+        client = self.make_client(make_folder("f1", "root", "/"), [])
+        with tempfile.TemporaryDirectory() as tmp:
+            local = os.path.join(tmp, "data.txt")
+            with open(local, "w") as f:
+                f.write("x")
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.create.side_effect = FileNotFoundError(local)
+                with self.assertRaises(UnexpectedException):
+                    Uploader(client).upload(local, "myremote:/mylab/", False, False)
+
+    def test_a_failed_upload_names_the_file(self):
+        client = self.make_client(make_folder("f1", "root", "/"), [])
+        with tempfile.TemporaryDirectory() as tmp:
+            local = os.path.join(tmp, "data.txt")
+            with open(local, "w") as f:
+                f.write("x")
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
+                with patch("sys.stdout", new=StringIO()) as fake_out:
+                    with self.assertRaises(UnexpectedException):
+                        Uploader(client).upload(local, "myremote:/mylab/", False, False)
+                    self.assertIn(local, fake_out.getvalue())
+
+    def test_an_upload_that_worked_stays_quiet(self):
+        client = self.make_client(make_folder("f1", "root", "/"), [])
+        with tempfile.TemporaryDirectory() as tmp:
+            local = os.path.join(tmp, "data.txt")
+            with open(local, "w") as f:
+                f.write("x")
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.create.return_value = "new-id"
+                Uploader(client).upload(local, "myremote:/mylab/", False, False)
+                files_api_class.return_value.create.assert_called_once()
+
+
+class TestDownloadReportsFailure(unittest.TestCase):
+    def make_client(self, remote_path_parts, folder: Folder, files_by_folder: dict[str, list[File]]) -> MagicMock:
+        client = MagicMock()
+        client.is_doi.return_value = False
+        client.parse_remote_host_with_path.return_value = remote_path_parts
+        client.find_laboratory.return_value = LABORATORY
+        client.find_folder.return_value = folder
+        client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
+        return client
+
+    def test_a_single_file_that_failed_is_raised_to_the_caller(self):
+        parent = make_folder("p1", "root", "/")
+        client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
+        with tempfile.TemporaryDirectory() as tmp:
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.download.side_effect = OSError("Permission denied.")
+                with self.assertRaises(UnexpectedException):
+                    Downloader(client).download("myremote:/mylab/data.txt", tmp)
+
+    def test_a_single_file_that_arrived_stays_quiet(self):
+        parent = make_folder("p1", "root", "/")
+        client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
+        with tempfile.TemporaryDirectory() as tmp:
+            with patch("mdrsclient.transfer.FilesApi") as files_api_class:
+                files_api_class.return_value.download.return_value = True
+                Downloader(client).download("myremote:/mylab/data.txt", tmp)
+                files_api_class.return_value.download.assert_called_once()
+
+    def test_a_failed_file_does_not_abandon_the_remaining_sub_folders(self):
+        parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
+        folders = {
+            "f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
+            "fa": make_folder("fa", "a", "/root/a/"),
+            "fb": make_folder("fb", "b", "/root/b/"),
+        }
+        files_by_folder = {
+            "p1": [],
+            "f1": [make_file("bad", "bad.txt")],
+            "fa": [make_file("good1", "good1.txt")],
+            "fb": [make_file("good2", "good2.txt")],
+        }
+        client = self.make_client(("myremote", "mylab", "/root"), parent, files_by_folder)
+        attempted: list[str] = []
+
+        def download(file: File, path: str) -> bool:
+            attempted.append(file.name)
+            if file.name == "bad.txt":
+                raise OSError("Permission denied.")
+            return True
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with (
+                patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
+                patch("mdrsclient.transfer.FilesApi") as files_api_class,
+            ):
+                folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
+                files_api_class.return_value.download.side_effect = download
+                with self.assertRaises(UnexpectedException):
+                    Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
+
+        self.assertEqual(sorted(attempted), ["bad.txt", "good1.txt", "good2.txt"])
+
+
+class TestRecursiveDownloadResilience(unittest.TestCase):
+    """A folder the client cannot prepare locally must not abandon its siblings."""
+
+    def test_a_folder_that_cannot_be_created_does_not_stop_the_walk(self):
+        parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
+        folders = {
+            "f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
+            "fa": make_folder("fa", "a", "/root/a/"),
+            "fb": make_folder("fb", "b", "/root/b/"),
+        }
+        files_by_folder = {"p1": [], "f1": [], "fa": [make_file("good1", "good1.txt")], "fb": []}
+        client = MagicMock()
+        client.is_doi.return_value = False
+        client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/root")
+        client.find_laboratory.return_value = LABORATORY
+        client.find_folder.return_value = parent
+        client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
+
+        real_makedirs = os.makedirs
+
+        def makedirs(path, *args, **kwargs):
+            if os.path.basename(path) == "b":
+                raise PermissionError("Permission denied")
+            return real_makedirs(path, *args, **kwargs)
+
+        attempted: list[str] = []
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with (
+                patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
+                patch("mdrsclient.transfer.FilesApi") as files_api_class,
+                patch("mdrsclient.transfer.os.makedirs", side_effect=makedirs),
+            ):
+                folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
+                files_api_class.return_value.download.side_effect = lambda file, path: attempted.append(file.name)
+                with self.assertRaises(UnexpectedException):
+                    Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
+
+        self.assertEqual(attempted, ["good1.txt"])
+
+
+class TestFileDownloadPermission(unittest.TestCase):
+    """A file the client could not write is a failure, not a line of successful output."""
+
+    def make_connection(self, chunks) -> MagicMock:
+        connection = MagicMock()
+        connection.token = None
+        response = MagicMock()
+        response.status_code = 200
+        response.iter_content.return_value = chunks
+        connection.get.return_value = response
+        return connection
+
+    def test_a_download_that_cannot_be_written_is_reported(self):
+        connection = self.make_connection([b"payload"])
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "precious.dat")
+            with open(path, "wb") as f:
+                f.write(b"do not touch")
+            os.chmod(path, 0o444)
+            try:
+                with open(path, "r+b"):
+                    self.skipTest("running with rights that ignore the file mode")
+            except OSError:
+                pass
+
+            with self.assertRaises(UnexpectedException):
+                FilesApi(connection).download(make_file("x1", "precious.dat"), path)
+
+            os.chmod(path, 0o644)
+            with open(path, "rb") as f:
+                self.assertEqual(f.read(), b"do not touch")
+            connection.get.assert_not_called()
+
+    def test_an_interrupted_download_leaves_the_existing_file_intact(self):
+        def chunks():
+            yield b"half a file"
+            raise ConnectionError("connection reset")
+
+        connection = self.make_connection(chunks())
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "existing.dat")
+            with open(path, "wb") as f:
+                f.write(b"the copy already here")
+
+            with self.assertRaises(ConnectionError):
+                FilesApi(connection).download(make_file("x1", "existing.dat"), path)
+
+            with open(path, "rb") as f:
+                self.assertEqual(f.read(), b"the copy already here")
+            self.assertEqual(os.listdir(tmp), ["existing.dat"])
+
+    def test_a_finished_download_replaces_the_destination(self):
+        connection = self.make_connection([b"new ", b"contents"])
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "existing.dat")
+            with open(path, "wb") as f:
+                f.write(b"old")
+
+            FilesApi(connection).download(make_file("x1", "existing.dat"), path)
+
+            with open(path, "rb") as f:
+                self.assertEqual(f.read(), b"new contents")
+            self.assertEqual(os.listdir(tmp), ["existing.dat"])
+
+
+if __name__ == "__main__":
+    unittest.main()