Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
8e03f7a7f4
|
|||
|
d59a150b4f
|
|||
|
7f6d496654
|
|||
|
b95fc0cd7d
|
|||
|
8ce9e09e69
|
+143
@@ -0,0 +1,143 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [1.3.18] - 2026-07-02
|
||||
|
||||
### Added
|
||||
- Added a comprehensive unit test suite in `tests/test_commands.py` checking registration, parsing, and execution flow of all 16 commands.
|
||||
|
||||
### Refactored
|
||||
- Abstracted configuration storage (introducing `ConfigInterface`, `InMemoryConfig`, and updating `ConfigFile`) to enable dependency injection.
|
||||
- Modularized transfer operations (upload and download) to decouple them from the service layer.
|
||||
- Decoupled commands from direct file system configurations and migrated all subcommands to use abstract config classes.
|
||||
|
||||
### Changed
|
||||
- Upgraded dependencies including `pydantic-settings` to `2.14.2` and `pyright` to `1.1.411`.
|
||||
|
||||
### Fixed
|
||||
- Fixed duplicate `__all__` definitions in package initialization file `mdrsclient/__init__.py` that caused `__version__` export to be overwritten.
|
||||
|
||||
## [1.3.17] - 2026-07-02
|
||||
|
||||
### Refactored
|
||||
- Decoupled core logic from CLI interface and introduced `MdrsClient` service layer to improve library portability.
|
||||
- Migrated all CLI commands to utilize `MdrsClient` for execution.
|
||||
|
||||
### Added
|
||||
- Abstract authentication state using `CacheInterface` and `InMemoryCache`.
|
||||
|
||||
## [1.3.16] - 2026-06-12
|
||||
|
||||
### Fixed
|
||||
- Retrieve the full `Folder` object from `FoldersApi` instead of using the `FolderSimple` returned by `find_sub_folder` when resolving DOI subfolders. This fixes a type checker error under the upgraded pyright and avoids a potential AttributeError at runtime due to `FolderSimple` lacking the `path` attribute.
|
||||
|
||||
### Changed
|
||||
- Upgraded dependencies and bumped version to 1.3.16 in pyproject.toml.
|
||||
|
||||
## [1.3.15] - 2026-05-01
|
||||
|
||||
### Fixed
|
||||
- Apply NFC normalization to filenames and folder names sent to the server.
|
||||
|
||||
## [1.3.14] - 2026-04-17
|
||||
|
||||
### Changed
|
||||
- Simplified `config list` command (removed `-l`/`--long` option, always display URL).
|
||||
- Renamed `--quick` option to `--quiet` for `ls` subcommand.
|
||||
|
||||
### Added
|
||||
- Added subcommand aliases for config commands (e.g. `ls` alias for list, `rm` alias for delete).
|
||||
- Added `version` command.
|
||||
|
||||
## [1.3.13] - 2025-07-02
|
||||
|
||||
### Fixed
|
||||
- Fixed pagination logic for the `file.list` API.
|
||||
|
||||
## [1.3.12] - 2025-05-20
|
||||
|
||||
### Fixed
|
||||
- Fixed bug where file downloading was skipped incorrectly when `-s`/`--skip-if-file-exists` option was present.
|
||||
|
||||
## [1.3.11] - 2025-01-21
|
||||
|
||||
### Fixed
|
||||
- Follow-up fixes for User API specification changes.
|
||||
|
||||
## [1.3.10] - 2024-12-23
|
||||
|
||||
### Added
|
||||
- Delete broken files and show a summary when a file download fails.
|
||||
|
||||
### Changed
|
||||
- Updated dependency libraries.
|
||||
|
||||
## [1.3.9] - 2024-10-23
|
||||
|
||||
### Fixed
|
||||
- Fixed compatibility with Python 3.10.
|
||||
|
||||
## [1.3.8] - 2024-09-18
|
||||
|
||||
### Added
|
||||
- Implemented `-s`/`--skip-if-file-exists` option for `download` command.
|
||||
|
||||
### Fixed
|
||||
- Added exception handling for unexpected responses from the server.
|
||||
|
||||
## [1.3.7] - 2024-07-22
|
||||
|
||||
### Added
|
||||
- Implemented `--exclude` argument for download subcommand.
|
||||
|
||||
## [1.3.6] - 2024-07-08
|
||||
|
||||
### Added
|
||||
- Support cancelling recursive downloads if downloading some files fails.
|
||||
|
||||
## [1.3.5] - 2024-07-08
|
||||
|
||||
### Added
|
||||
- Added authorization token validation checks for file download operations.
|
||||
|
||||
### Removed
|
||||
- Removed unnecessary debug code.
|
||||
|
||||
## [1.3.4] - 2024-07-04
|
||||
|
||||
### Added
|
||||
- Added some aliases for config sub command.
|
||||
|
||||
### Fixed
|
||||
- Fixed bug when uploading large files.
|
||||
|
||||
## [1.3.3] - 2024-02-13
|
||||
|
||||
### Added
|
||||
- Implemented `-s`/`--skip-if-file-exists` option for `upload` command.
|
||||
|
||||
## [1.3.2] - 2024-02-09
|
||||
|
||||
### Added
|
||||
- Added `-u` and `-p` options to login command.
|
||||
|
||||
## [1.3.1] - 2023-12-20
|
||||
|
||||
### Fixed
|
||||
- Fixed bug to resolve local files for recursive file upload.
|
||||
|
||||
## [1.3.0] - 2023-12-18
|
||||
|
||||
### Changed
|
||||
- Removed debug comments.
|
||||
|
||||
## [1.2.0] - 2023-10-04
|
||||
|
||||
### Changed
|
||||
- Follow-up recent specification changes about folder access level.
|
||||
|
||||
## [1.1.1] - 2023-07-26
|
||||
|
||||
### Changed
|
||||
- Set destination folder name using name attribute of folder copy API.
|
||||
@@ -213,15 +213,21 @@ You can also use this package as a Python library to programmatically interact w
|
||||
```python
|
||||
from mdrsclient.client import MdrsClient
|
||||
from mdrsclient.cache import InMemoryCache
|
||||
from mdrsclient.config import InMemoryConfig
|
||||
|
||||
# 1. Setup in-memory configuration and cache to avoid local state files (e.g., config.ini, cache/*.json)
|
||||
config = InMemoryConfig("neurodata")
|
||||
config.url = "https://neurodata.riken.jp/api"
|
||||
|
||||
# 1. Setup client with an in-memory cache to avoid local `.mdrsclient` state files
|
||||
cache = InMemoryCache()
|
||||
client = MdrsClient.from_remote("neurodata", cache=cache)
|
||||
|
||||
# 2. Login to the remote server
|
||||
# 2. Initialize client with custom configuration and cache
|
||||
client = MdrsClient.from_remote("neurodata", cache=cache, config=config)
|
||||
|
||||
# 3. Login to the remote server
|
||||
client.login("username", "password")
|
||||
|
||||
# 3. Use service methods
|
||||
# 4. Use service methods
|
||||
labs = client.get_laboratories()
|
||||
metadata = client.metadata("neurodata:/NIU/Repository/")
|
||||
|
||||
@@ -230,3 +236,15 @@ client.upload("/path/to/local/data", "neurodata:/NIU/Repository/TEST/", is_recur
|
||||
client.download("neurodata:/NIU/Repository/TEST/data", "/path/to/local", is_recursive=True)
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
You can run the unit test suite using the standard library `unittest` discover runner:
|
||||
|
||||
```shell
|
||||
.venv/bin/python -m unittest discover tests
|
||||
```
|
||||
|
||||
## Changelog
|
||||
|
||||
See [CHANGELOG.md](./CHANGELOG.md) for the full change history.
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
from mdrsclient.__version__ import __version__
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
__all__ = ["MdrsClient"]
|
||||
__all__ = ["__version__", "MdrsClient"]
|
||||
|
||||
+2
-2
@@ -66,7 +66,7 @@ class CacheInterface(Protocol):
|
||||
def laboratories(self, laboratories: Laboratories) -> None: ...
|
||||
|
||||
|
||||
class InMemoryCache:
|
||||
class InMemoryCache(CacheInterface):
|
||||
def __init__(self) -> None:
|
||||
self.__data = CacheData()
|
||||
|
||||
@@ -105,7 +105,7 @@ class InMemoryCache:
|
||||
self.__data.laboratories = laboratories
|
||||
|
||||
|
||||
class CacheFile:
|
||||
class CacheFile(CacheInterface):
|
||||
__serial: int
|
||||
__cache_dir: str
|
||||
__cache_file: str
|
||||
|
||||
+65
-92
@@ -1,69 +1,50 @@
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from mdrsclient.api import DoiApi, FilesApi, FoldersApi, LaboratoriesApi, UsersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.cache import CacheInterface
|
||||
from mdrsclient.config import ConfigInterface
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, MDRSException, UnauthorizedException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, Laboratory, Token, User
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
from mdrsclient.services import MdrsService
|
||||
|
||||
|
||||
class MdrsClient:
|
||||
class MdrsClient(MdrsService):
|
||||
"""Service layer client for MDRS."""
|
||||
|
||||
def __init__(self, connection: MDRSConnection):
|
||||
self.connection = connection
|
||||
def __init__(self, connection: MDRSConnection, config_class: type[ConfigInterface] | None = None):
|
||||
super().__init__(connection, config_class)
|
||||
|
||||
@classmethod
|
||||
def from_remote(cls, remote: str) -> "MdrsClient":
|
||||
return cls(BaseCommand._create_connection(remote))
|
||||
|
||||
def login(self, username: str, password: str) -> tuple[Token, User]:
|
||||
user_api = UsersApi(self.connection)
|
||||
token = user_api.token(username, password)
|
||||
self.connection.token = token
|
||||
user = user_api.current()
|
||||
self.connection.user = user
|
||||
return token, user
|
||||
|
||||
def logout(self) -> None:
|
||||
self.connection.logout()
|
||||
|
||||
def whoami(self) -> User:
|
||||
user_api = UsersApi(self.connection)
|
||||
return user_api.current()
|
||||
|
||||
def get_laboratories(self) -> list[Laboratory]:
|
||||
laboratory_api = LaboratoriesApi(self.connection)
|
||||
labs = laboratory_api.list()
|
||||
self.connection.laboratories = labs
|
||||
return list(labs)
|
||||
def from_remote(
|
||||
cls, remote: str, cache: CacheInterface | None = None, config: ConfigInterface | None = None
|
||||
) -> "MdrsClient":
|
||||
return cls(cls.create_connection(remote, cache, config))
|
||||
|
||||
def mkdir(self, remote_path: str) -> None:
|
||||
remote, laboratory_name, r_path = BaseCommand._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
laboratory = BaseCommand._find_laboratory(self.connection, laboratory_name)
|
||||
parent_folder = BaseCommand._find_folder(self.connection, laboratory, r_dirname)
|
||||
files = BaseCommand._find_files(self.connection, parent_folder.id)
|
||||
laboratory = self.find_laboratory(laboratory_name)
|
||||
parent_folder = self.find_folder(laboratory, r_dirname)
|
||||
files = self.find_files(parent_folder.id)
|
||||
if parent_folder.find_sub_folder(r_basename) is not None or find_file(files, r_basename) is not None:
|
||||
raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
|
||||
folder_api = FoldersApi(self.connection)
|
||||
folder_api.create(normalize("NFC", r_basename), parent_folder.id)
|
||||
|
||||
def rm(self, remote_path: str, is_recursive: bool = False) -> None:
|
||||
remote, laboratory_name, r_path = BaseCommand._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
laboratory = BaseCommand._find_laboratory(self.connection, laboratory_name)
|
||||
parent_folder = BaseCommand._find_folder(self.connection, laboratory, r_dirname)
|
||||
parent_files = BaseCommand._find_files(self.connection, parent_folder.id)
|
||||
laboratory = self.find_laboratory(laboratory_name)
|
||||
parent_folder = self.find_folder(laboratory, r_dirname)
|
||||
parent_files = self.find_files(parent_folder.id)
|
||||
file = find_file(parent_files, r_basename)
|
||||
if file is not None:
|
||||
file_api = FilesApi(self.connection)
|
||||
@@ -78,13 +59,13 @@ class MdrsClient:
|
||||
folder_api.destroy(folder.id, True)
|
||||
|
||||
def ls(self, remote_path: str, password: str | None = None) -> tuple[Folder, list[File]]:
|
||||
folder, laboratory = BaseCommand._resolve_folder(self.connection, remote_path, password)
|
||||
files = BaseCommand._find_files(self.connection, folder.id)
|
||||
folder, laboratory = self.resolve_folder(remote_path, password)
|
||||
files = self.find_files(folder.id)
|
||||
return folder, files
|
||||
|
||||
def cp(self, src_path: str, dest_path: str, is_recursive: bool = False) -> None:
|
||||
s_remote, s_laboratory_name, s_path = BaseCommand._parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = BaseCommand._parse_remote_host_with_path(dest_path)
|
||||
s_remote, s_laboratory_name, s_path = self.parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = self.parse_remote_host_with_path(dest_path)
|
||||
if s_remote != d_remote:
|
||||
raise IllegalArgumentException("Remote host mismatched.")
|
||||
if s_laboratory_name != d_laboratory_name:
|
||||
@@ -98,11 +79,11 @@ class MdrsClient:
|
||||
else:
|
||||
d_dirname = os.path.dirname(d_path)
|
||||
d_basename = os.path.basename(d_path)
|
||||
laboratory = BaseCommand._find_laboratory(self.connection, s_laboratory_name)
|
||||
s_parent_folder = BaseCommand._find_folder(self.connection, laboratory, s_dirname)
|
||||
s_parent_files = BaseCommand._find_files(self.connection, s_parent_folder.id)
|
||||
d_parent_folder = BaseCommand._find_folder(self.connection, laboratory, d_dirname)
|
||||
d_parent_files = BaseCommand._find_files(self.connection, d_parent_folder.id)
|
||||
laboratory = self.find_laboratory(s_laboratory_name)
|
||||
s_parent_folder = self.find_folder(laboratory, s_dirname)
|
||||
s_parent_files = self.find_files(s_parent_folder.id)
|
||||
d_parent_folder = self.find_folder(laboratory, d_dirname)
|
||||
d_parent_files = self.find_files(d_parent_folder.id)
|
||||
s_file = find_file(s_parent_files, s_basename)
|
||||
if s_file is not None:
|
||||
d_file = find_file(d_parent_files, d_basename)
|
||||
@@ -132,8 +113,8 @@ class MdrsClient:
|
||||
folder_api.copy(s_folder, d_parent_folder.id, normalize("NFC", d_basename))
|
||||
|
||||
def mv(self, src_path: str, dest_path: str) -> None:
|
||||
s_remote, s_laboratory_name, s_path = BaseCommand._parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = BaseCommand._parse_remote_host_with_path(dest_path)
|
||||
s_remote, s_laboratory_name, s_path = self.parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = self.parse_remote_host_with_path(dest_path)
|
||||
if s_remote != d_remote:
|
||||
raise IllegalArgumentException("Remote host mismatched.")
|
||||
if s_laboratory_name != d_laboratory_name:
|
||||
@@ -147,11 +128,11 @@ class MdrsClient:
|
||||
else:
|
||||
d_dirname = os.path.dirname(d_path)
|
||||
d_basename = os.path.basename(d_path)
|
||||
laboratory = BaseCommand._find_laboratory(self.connection, s_laboratory_name)
|
||||
s_parent_folder = BaseCommand._find_folder(self.connection, laboratory, s_dirname)
|
||||
s_parent_files = BaseCommand._find_files(self.connection, s_parent_folder.id)
|
||||
d_parent_folder = BaseCommand._find_folder(self.connection, laboratory, d_dirname)
|
||||
d_parent_files = BaseCommand._find_files(self.connection, d_parent_folder.id)
|
||||
laboratory = self.find_laboratory(s_laboratory_name)
|
||||
s_parent_folder = self.find_folder(laboratory, s_dirname)
|
||||
s_parent_files = self.find_files(s_parent_folder.id)
|
||||
d_parent_folder = self.find_folder(laboratory, d_dirname)
|
||||
d_parent_files = self.find_files(d_parent_folder.id)
|
||||
s_file = find_file(s_parent_files, s_basename)
|
||||
if s_file is not None:
|
||||
d_file = find_file(d_parent_files, d_basename)
|
||||
@@ -181,36 +162,34 @@ class MdrsClient:
|
||||
def chacl(
|
||||
self, remote_path: str, access_level: int, is_recursive: bool = False, password: str | None = None
|
||||
) -> None:
|
||||
remote, laboratory_name, r_path = BaseCommand._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = self.parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
laboratory = BaseCommand._find_laboratory(self.connection, laboratory_name)
|
||||
folder = BaseCommand._find_folder(self.connection, laboratory, r_path)
|
||||
laboratory = self.find_laboratory(laboratory_name)
|
||||
folder = self.find_folder(laboratory, r_path)
|
||||
folder_api = FoldersApi(self.connection)
|
||||
folder_api.acl(folder.id, access_level, is_recursive, password)
|
||||
|
||||
def metadata(self, remote_path: str, password: str | None = None) -> dict:
|
||||
folder, laboratory = BaseCommand._resolve_folder(self.connection, remote_path, password)
|
||||
folder, laboratory = self.resolve_folder(remote_path, password)
|
||||
folder_api = FoldersApi(self.connection)
|
||||
return folder_api.metadata(folder.id)
|
||||
|
||||
def file_metadata(self, remote_path: str, password: str | None = None) -> dict:
|
||||
folder, laboratory, r_basename = BaseCommand._resolve_file(self.connection, remote_path, password)
|
||||
files = BaseCommand._find_files(self.connection, folder.id)
|
||||
folder, laboratory, r_basename = self.resolve_file(remote_path, password)
|
||||
files = self.find_files(folder.id)
|
||||
file = find_file(files, r_basename)
|
||||
if file is None:
|
||||
raise IllegalArgumentException(f"File `{r_basename}` not found.")
|
||||
file_api = FilesApi(self.connection)
|
||||
return file_api.metadata(file)
|
||||
|
||||
def _create_connection(self, remote: str):
|
||||
return self.connection
|
||||
|
||||
def upload(
|
||||
self, local_path: str, remote_path: str, is_recursive: bool = False, is_skip_if_exists: bool = False
|
||||
) -> None:
|
||||
from mdrsclient.commands.upload import UploadCommand
|
||||
from mdrsclient.transfer import Uploader
|
||||
|
||||
UploadCommand._upload_logic(self.connection, local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||
uploader = Uploader(self)
|
||||
uploader.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||
|
||||
def download(
|
||||
self,
|
||||
@@ -221,23 +200,10 @@ class MdrsClient:
|
||||
password: str | None = None,
|
||||
excludes: list[str] | None = None,
|
||||
) -> None:
|
||||
from mdrsclient.commands.download import DownloadCommand
|
||||
from mdrsclient.transfer import Downloader
|
||||
|
||||
DownloadCommand._download_logic(
|
||||
self.connection, remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes or []
|
||||
)
|
||||
|
||||
def ls_command(
|
||||
self,
|
||||
remote_path: str,
|
||||
password: str | None = None,
|
||||
is_json: bool = False,
|
||||
is_recursive: bool = False,
|
||||
is_quiet: bool = False,
|
||||
) -> None:
|
||||
from mdrsclient.commands.ls import LsCommand
|
||||
|
||||
LsCommand._ls_logic(self.connection, remote_path, password, is_json, is_recursive, is_quiet)
|
||||
downloader = Downloader(self)
|
||||
downloader.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
|
||||
|
||||
def version(self) -> str:
|
||||
from mdrsclient.__version__ import __version__
|
||||
@@ -245,22 +211,29 @@ class MdrsClient:
|
||||
return f"mdrs {__version__}"
|
||||
|
||||
def config_create(self, remote: str, url: str) -> None:
|
||||
from mdrsclient.commands.config import ConfigCommand
|
||||
|
||||
ConfigCommand.create(remote, url)
|
||||
remote = self.parse_remote_host(remote)
|
||||
config = self.config_class(remote)
|
||||
if config.url is not None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
|
||||
else:
|
||||
config.url = url
|
||||
|
||||
def config_update(self, remote: str, url: str) -> None:
|
||||
from mdrsclient.commands.config import ConfigCommand
|
||||
|
||||
ConfigCommand.update(remote, url)
|
||||
remote = self.parse_remote_host(remote)
|
||||
config = self.config_class(remote)
|
||||
if config.url is None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
|
||||
else:
|
||||
config.url = url
|
||||
|
||||
def config_list(self) -> list:
|
||||
from mdrsclient.config import ConfigFile
|
||||
|
||||
config = ConfigFile("")
|
||||
config = self.config_class("")
|
||||
return config.list()
|
||||
|
||||
def config_delete(self, remote: str) -> None:
|
||||
from mdrsclient.commands.config import ConfigCommand
|
||||
|
||||
ConfigCommand.delete(remote)
|
||||
remote = self.parse_remote_host(remote)
|
||||
config = self.config_class(remote)
|
||||
if config.url is None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
|
||||
else:
|
||||
del config.url
|
||||
|
||||
+1
-257
@@ -1,20 +1,7 @@
|
||||
import os
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from mdrsclient.api import DoiApi, FilesApi, FoldersApi, LaboratoriesApi
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import (
|
||||
IllegalArgumentException,
|
||||
MissingConfigurationException,
|
||||
UnauthorizedException,
|
||||
UnexpectedException,
|
||||
)
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.utils import page_num_from_url
|
||||
from mdrsclient.exceptions import UnexpectedException
|
||||
|
||||
|
||||
class BaseCommand(ABC):
|
||||
@@ -22,246 +9,3 @@ class BaseCommand(ABC):
|
||||
@abstractmethod
|
||||
def register(cls, parsers: Any) -> None:
|
||||
raise UnexpectedException("Not implemented.")
|
||||
|
||||
@classmethod
|
||||
def _create_connection(cls, remote: str) -> MDRSConnection:
|
||||
config = ConfigFile(remote)
|
||||
if config.url is None:
|
||||
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
|
||||
return MDRSConnection(config.remote, config.url)
|
||||
|
||||
@classmethod
|
||||
def _find_laboratory(cls, connection: MDRSConnection, name: str) -> Laboratory:
|
||||
if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired:
|
||||
laboratory_api = LaboratoriesApi(connection)
|
||||
connection.laboratories = laboratory_api.list()
|
||||
laboratory = connection.laboratories.find_by_name(name)
|
||||
if laboratory is None:
|
||||
raise IllegalArgumentException(f"Laboratory `{name}` not found.")
|
||||
return laboratory
|
||||
|
||||
@classmethod
|
||||
def _find_folder(
|
||||
cls, connection: MDRSConnection, laboratory: Laboratory, path: str, password: str | None = None
|
||||
) -> Folder:
|
||||
folder_api = FoldersApi(connection)
|
||||
folders = folder_api.list(laboratory.id, normalize("NFC", path))
|
||||
if len(folders) != 1:
|
||||
raise UnexpectedException(f"Folder `{path}` not found.")
|
||||
if folders[0].lock:
|
||||
if password is None:
|
||||
raise UnauthorizedException(f"Folder `{path}` is locked.")
|
||||
folder_api.auth(folders[0].id, password)
|
||||
return folder_api.retrieve(folders[0].id)
|
||||
|
||||
@classmethod
|
||||
def _find_files(cls, connection: MDRSConnection, folder_id: str) -> list[File]:
|
||||
files_api = FilesApi(connection)
|
||||
page = 1
|
||||
results_file = []
|
||||
while page:
|
||||
result = files_api.list(folder_id, page)
|
||||
results_file.extend(result.results)
|
||||
page = 0
|
||||
if result.next:
|
||||
page = page_num_from_url(result.next)
|
||||
return results_file
|
||||
|
||||
@classmethod
|
||||
def _parse_remote_host(cls, path: str) -> str:
|
||||
path_array = path.split(":")
|
||||
remote_host = path_array[0]
|
||||
if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2:
|
||||
raise IllegalArgumentException("Invalid remote host")
|
||||
return remote_host
|
||||
|
||||
@classmethod
|
||||
def _parse_remote_host_with_path(cls, path: str) -> tuple[str, str, str]:
|
||||
path = re.sub(r"//+|/\./+|/\.$", "/", path)
|
||||
if re.search(r"/\.\./|/\.\.$", path) is not None:
|
||||
raise IllegalArgumentException("Path traversal found.")
|
||||
path_array = path.split(":")
|
||||
if len(path_array) != 2:
|
||||
raise IllegalArgumentException("Invalid remote host.")
|
||||
remote_host = path_array[0]
|
||||
folder_array = path_array[1].split("/")
|
||||
is_absolute_path = folder_array[0] == ""
|
||||
if not is_absolute_path:
|
||||
raise IllegalArgumentException("Must be absolute paths.")
|
||||
del folder_array[0]
|
||||
if len(folder_array) == 0:
|
||||
laboratory = ""
|
||||
folder = ""
|
||||
else:
|
||||
laboratory = folder_array.pop(0)
|
||||
folder = "/" + "/".join(folder_array)
|
||||
return (remote_host, laboratory, folder)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# DOI helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _is_doi(path_component: str) -> bool:
|
||||
"""Return True if path_component looks like a DOI string.
|
||||
|
||||
A DOI is recognised as a string that starts with ``10.`` and
|
||||
contains a ``/``.
|
||||
"""
|
||||
return path_component.startswith("10.") and "/" in path_component
|
||||
|
||||
@staticmethod
|
||||
def _doi_suffix_id(doi: str) -> str:
|
||||
"""Extract the internal system ID from a full DOI string.
|
||||
|
||||
MDRS uses the segment after the last ``.`` in the suffix (the part
|
||||
after the ``/``) as its identifier.
|
||||
Example: ``10.xxxx/prefix.20230511-001`` → ``20230511-001``.
|
||||
If there is no ``.`` in the suffix, the whole suffix is returned.
|
||||
Trailing slashes are stripped before processing.
|
||||
"""
|
||||
# Strip any trailing slash first.
|
||||
doi = doi.rstrip("/")
|
||||
slash_pos = doi.find("/")
|
||||
if slash_pos == -1:
|
||||
return doi
|
||||
suffix = doi[slash_pos + 1 :]
|
||||
dot_pos = suffix.rfind(".")
|
||||
return suffix[dot_pos + 1 :] if dot_pos != -1 else suffix
|
||||
|
||||
@staticmethod
|
||||
def _split_doi_and_subpath(doi_with_path: str) -> tuple[str, str]:
|
||||
"""Split a DOI-with-optional-path string into (doi, subpath)."""
|
||||
# Find the first '/' that separates registrant from suffix.
|
||||
first_slash = doi_with_path.find("/")
|
||||
if first_slash != -1:
|
||||
after_suffix_start = first_slash + 1
|
||||
after_first = doi_with_path[after_suffix_start:]
|
||||
# Find the next '/' inside the suffix portion — this starts the subpath.
|
||||
second_slash = after_first.find("/")
|
||||
if second_slash != -1:
|
||||
doi_end = after_suffix_start + second_slash
|
||||
doi = doi_with_path[:doi_end]
|
||||
subpath = doi_with_path[doi_end:] # begins with "/"
|
||||
# Treat a bare trailing slash as no subpath (root of DOI folder).
|
||||
if subpath == "/":
|
||||
return (doi, "")
|
||||
else:
|
||||
return (doi, subpath)
|
||||
else:
|
||||
# No second slash — the whole string is the DOI, no subpath.
|
||||
return (doi_with_path, "")
|
||||
else:
|
||||
return (doi_with_path, "")
|
||||
|
||||
@classmethod
|
||||
def _parse_doi_remote_host(cls, path: str) -> tuple[str, str, str]:
|
||||
"""Parse ``remote:10.xxxx/prefix.ID[/optional/sub/path]`` into ``(remote, doi, subpath)``."""
|
||||
parts = path.split(":", 1)
|
||||
if len(parts) != 2:
|
||||
raise IllegalArgumentException("remote_path must be in the form 'remote:10.xxxx/prefix.ID'")
|
||||
remote, doi_with_path = parts
|
||||
if not cls._is_doi(doi_with_path):
|
||||
raise IllegalArgumentException(
|
||||
f"Path `{doi_with_path}` does not look like a DOI (must start with '10.' and contain '/')."
|
||||
)
|
||||
doi, subpath = cls._split_doi_and_subpath(doi_with_path)
|
||||
return (remote, doi, subpath)
|
||||
|
||||
@classmethod
|
||||
def _find_folder_by_doi(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
doi: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory]:
|
||||
"""Resolve a DOI to a (Folder, Laboratory) pair.
|
||||
|
||||
Calls GET v3/doi/{id}/ to look up the folder ID, retrieves the full
|
||||
folder detail (which carries ``laboratory_id``), and resolves the
|
||||
laboratory from that field.
|
||||
"""
|
||||
doi_clean = doi.rstrip("/")
|
||||
doi_id = cls._doi_suffix_id(doi_clean)
|
||||
doi_api = DoiApi(connection)
|
||||
doi_resp = doi_api.retrieve(doi_id)
|
||||
|
||||
# Verify the returned DOI matches the one supplied (case-insensitive).
|
||||
returned_doi = doi_resp.doi.rstrip("/")
|
||||
if returned_doi.lower() != doi_clean.lower():
|
||||
raise IllegalArgumentException(
|
||||
f"DOI mismatch: requested `{doi_clean}` but server returned `{returned_doi}`."
|
||||
)
|
||||
|
||||
folder_api = FoldersApi(connection)
|
||||
|
||||
# Retrieve full folder detail directly by ID; laboratory_id is here.
|
||||
folder = folder_api.retrieve(doi_resp.folder_id)
|
||||
|
||||
if folder.lock:
|
||||
if password is None:
|
||||
raise UnauthorizedException(f"Folder for DOI `{doi_clean}` is locked.")
|
||||
folder_api.auth(doi_resp.folder.id, password)
|
||||
|
||||
# Resolve laboratory using laboratory_id from the full folder detail.
|
||||
lab_api = LaboratoriesApi(connection)
|
||||
labs = lab_api.list()
|
||||
lab = labs.find_by_id(folder.laboratory_id)
|
||||
if lab is None:
|
||||
raise UnexpectedException(f"Laboratory with id {folder.laboratory_id} not found.")
|
||||
|
||||
connection.laboratories = labs
|
||||
return (folder, lab)
|
||||
|
||||
@classmethod
|
||||
def _resolve_folder(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
remote_path: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory]:
|
||||
"""Resolve any remote path (normal or DOI) into a (Folder, Laboratory) pair."""
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
if not subpath:
|
||||
return (doi_folder, laboratory)
|
||||
else:
|
||||
abs_path = doi_folder.path.rstrip("/") + subpath
|
||||
folder = cls._find_folder(connection, laboratory, abs_path, password)
|
||||
return (folder, laboratory)
|
||||
else:
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_path, password)
|
||||
return (folder, laboratory)
|
||||
|
||||
@classmethod
|
||||
def _resolve_file(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
remote_path: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory, str]:
|
||||
"""Resolve a remote path pointing to a file into the parent Folder, its Laboratory, and the file's basename."""
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
raise IllegalArgumentException("DOI path must point to a file, not a folder.")
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
parent_folder = cls._find_folder(connection, laboratory, abs_path, password)
|
||||
return (parent_folder, laboratory, r_basename)
|
||||
else:
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
return (parent_folder, laboratory, r_basename)
|
||||
|
||||
@@ -2,8 +2,6 @@ from argparse import Namespace
|
||||
from typing import Any, Callable
|
||||
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
|
||||
|
||||
class ConfigCommand(BaseCommand):
|
||||
@@ -52,35 +50,6 @@ class ConfigCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def func_list(cls, args: Namespace) -> None:
|
||||
cls.list()
|
||||
|
||||
@classmethod
|
||||
def func_delete(cls, args: Namespace) -> None:
|
||||
remote = str(args.remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
MdrsClient(None).config_delete(remote)
|
||||
|
||||
@classmethod
|
||||
def create(cls, remote: str, url: str) -> None:
|
||||
remote = cls._parse_remote_host(remote)
|
||||
config = ConfigFile(remote)
|
||||
if config.url is not None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
|
||||
else:
|
||||
config.url = url
|
||||
|
||||
@classmethod
|
||||
def update(cls, remote: str, url: str) -> None:
|
||||
remote = cls._parse_remote_host(remote)
|
||||
config = ConfigFile(remote)
|
||||
if config.url is None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
|
||||
else:
|
||||
config.url = url
|
||||
|
||||
@classmethod
|
||||
def list(cls) -> None:
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
client = MdrsClient(None)
|
||||
@@ -88,10 +57,8 @@ class ConfigCommand(BaseCommand):
|
||||
print(f"{remote}:\t{url}")
|
||||
|
||||
@classmethod
|
||||
def delete(cls, remote: str) -> None:
|
||||
remote = cls._parse_remote_host(remote)
|
||||
config = ConfigFile(remote)
|
||||
if config.url is None:
|
||||
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
|
||||
else:
|
||||
del config.url
|
||||
def func_delete(cls, args: Namespace) -> None:
|
||||
remote = str(args.remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
MdrsClient(None).config_delete(remote)
|
||||
|
||||
@@ -1,30 +1,7 @@
|
||||
import os
|
||||
from argparse import Namespace
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadFileInfo:
|
||||
file: File
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
hasError: bool
|
||||
isSkipIfExists: bool
|
||||
files: list[DownloadFileInfo]
|
||||
|
||||
|
||||
class DownloadCommand(BaseCommand):
|
||||
@@ -37,7 +14,7 @@ class DownloadCommand(BaseCommand):
|
||||
download_parser.add_argument(
|
||||
"-s",
|
||||
"--skip-if-exists",
|
||||
help="skip the download if file is already downloaded and file size is the same",
|
||||
help="skip the download if file is already uploaded and file size is the same",
|
||||
action="store_true",
|
||||
)
|
||||
download_parser.add_argument(
|
||||
@@ -74,160 +51,3 @@ class DownloadCommand(BaseCommand):
|
||||
client = MdrsClient.from_remote(remote)
|
||||
client.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def _download_logic(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
remote_path: str,
|
||||
local_path: str,
|
||||
is_recursive: bool,
|
||||
is_skip_if_exists: bool,
|
||||
password: str | None,
|
||||
excludes: list[str],
|
||||
) -> None:
|
||||
# Detect DOI path: "remote:10.xxxx/prefix.ID[/optional/sub/path]"
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
folder = doi_folder
|
||||
is_folder = True
|
||||
else:
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_parent_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
r_parent_folder = cls._find_folder(connection, laboratory, abs_parent_path, password)
|
||||
r_parent_files = cls._find_files(connection, r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
return
|
||||
else:
|
||||
folder_simple = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder_simple is None:
|
||||
raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
|
||||
folder = FoldersApi(connection).retrieve(folder_simple.id)
|
||||
is_folder = True
|
||||
|
||||
# For a DOI target the whole folder is the download target.
|
||||
if not is_recursive:
|
||||
# Non-recursive: download only the files at the top level of the DOI folder.
|
||||
files = cls._find_files(connection, folder.id)
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
for file in files:
|
||||
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
l_path = os.path.join(l_dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
return
|
||||
folder_api = FoldersApi(connection)
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
return
|
||||
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
r_parent_files = cls._find_files(connection, r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
else:
|
||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder is None:
|
||||
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
folder_api = FoldersApi(connection)
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_pickup_recursive_files(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
folder_api: FoldersApi,
|
||||
laboratory: Laboratory,
|
||||
folder_id: str,
|
||||
basedir: str,
|
||||
excludes: list[str],
|
||||
is_skip_if_exists: bool,
|
||||
) -> None:
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
folder = folder_api.retrieve(folder_id)
|
||||
files = cls._find_files(connection, folder.id)
|
||||
dirname = os.path.join(basedir, folder.name)
|
||||
if cls.__check_excludes(excludes, laboratory, folder, None):
|
||||
return
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
print(dirname)
|
||||
for file in files:
|
||||
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
path = os.path.join(dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, path))
|
||||
cls.__multiple_download(connection, context)
|
||||
if context.hasError:
|
||||
raise UnexpectedException("Some files failed to download.")
|
||||
for sub_folder in folder.sub_folders:
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
|
||||
file_api = FilesApi(connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
results = pool.map(
|
||||
lambda x: cls.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
|
||||
)
|
||||
hasError = next(filter(lambda x: x is False, results), None)
|
||||
if hasError is not None:
|
||||
context.hasError = True
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||
if not is_skip_if_exists or not os.path.exists(info.path) or info.file.size != os.path.getsize(info.path):
|
||||
try:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
print(f"Failed: ${info.path}")
|
||||
if os.path.isfile(info.path):
|
||||
os.remove(info.path)
|
||||
return False
|
||||
print(info.path)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
|
||||
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
|
||||
return path in excludes
|
||||
|
||||
@@ -19,9 +19,9 @@ class LabsCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def labs(cls, remote: str) -> None:
|
||||
remote_host = cls._parse_remote_host(remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
remote_host = MdrsClient.parse_remote_host(remote)
|
||||
client = MdrsClient.from_remote(remote_host)
|
||||
laboratories = client.get_laboratories()
|
||||
label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
|
||||
|
||||
@@ -2,11 +2,7 @@ import getpass
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from mdrsclient.api import UsersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import MissingConfigurationException
|
||||
|
||||
|
||||
class LoginCommand(BaseCommand):
|
||||
@@ -27,9 +23,9 @@ class LoginCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def login(cls, remote: str, username: str, password: str) -> None:
|
||||
remote_host = cls._parse_remote_host(remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
remote_host = MdrsClient.parse_remote_host(remote)
|
||||
client = MdrsClient.from_remote(remote_host)
|
||||
client.login(username, password)
|
||||
print("Login Successful")
|
||||
|
||||
@@ -2,9 +2,6 @@ from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import MissingConfigurationException
|
||||
|
||||
|
||||
class LogoutCommand(BaseCommand):
|
||||
@@ -21,8 +18,8 @@ class LogoutCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def logout(cls, remote: str) -> None:
|
||||
remote_host = cls._parse_remote_host(remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
remote_host = MdrsClient.parse_remote_host(remote)
|
||||
client = MdrsClient.from_remote(remote_host)
|
||||
client.logout()
|
||||
|
||||
+20
-14
@@ -5,8 +5,8 @@ from typing import Any
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.client import MdrsClient
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import UnauthorizedException
|
||||
from mdrsclient.models import File, Folder, FolderSimple, Laboratory
|
||||
|
||||
@@ -19,7 +19,7 @@ class Config:
|
||||
@dataclass(config=Config)
|
||||
class LsCommandContext:
|
||||
prefix: str
|
||||
connection: MDRSConnection
|
||||
client: MdrsClient
|
||||
laboratory: Laboratory
|
||||
password: str
|
||||
is_json: bool
|
||||
@@ -58,21 +58,27 @@ class LsCommand(BaseCommand):
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
client = MdrsClient.from_remote(remote)
|
||||
client.ls_command(remote_path, password, is_json, is_recursive, is_quiet)
|
||||
cls._ls_logic(client, remote_path, password, is_json, is_recursive, is_quiet)
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def _ls_logic(
|
||||
cls, connection, remote_path: str, password: str | None, is_json: bool, is_recursive: bool, is_quiet: bool
|
||||
cls,
|
||||
client: MdrsClient,
|
||||
remote_path: str,
|
||||
password: str | None,
|
||||
is_json: bool,
|
||||
is_recursive: bool,
|
||||
is_quiet: bool,
|
||||
) -> None:
|
||||
remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
|
||||
|
||||
folder, laboratory = cls._resolve_folder(connection, remote_path, password)
|
||||
folder, laboratory = client.resolve_folder(remote_path, password)
|
||||
laboratory_name = laboratory.name
|
||||
files = cls._find_files(connection, folder.id)
|
||||
files = client.find_files(folder.id)
|
||||
context = LsCommandContext(
|
||||
f"{remote}:/{laboratory_name}",
|
||||
connection,
|
||||
client,
|
||||
laboratory,
|
||||
password if password is not None else "",
|
||||
is_json,
|
||||
@@ -102,7 +108,7 @@ class LsCommand(BaseCommand):
|
||||
for key in label.keys():
|
||||
length[key] = len(label[key]) if not context.is_quiet else 0
|
||||
for sub_folder in folder.sub_folders:
|
||||
sub_laboratory = context.connection.laboratories.find_by_id(sub_folder.laboratory_id)
|
||||
sub_laboratory = context.client.connection.laboratories.find_by_id(sub_folder.laboratory_id)
|
||||
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
|
||||
length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
|
||||
length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
|
||||
@@ -147,12 +153,12 @@ class LsCommand(BaseCommand):
|
||||
if context.is_recursive:
|
||||
print("")
|
||||
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
|
||||
folder_api = FoldersApi(context.connection)
|
||||
folder_api = FoldersApi(context.client.connection)
|
||||
try:
|
||||
if sub_folder.lock:
|
||||
folder_api.auth(sub_folder.id, context.password)
|
||||
folder = folder_api.retrieve(sub_folder.id)
|
||||
files = cls._find_files(context.connection, sub_folder.id)
|
||||
files = context.client.find_files(sub_folder.id)
|
||||
cls._ls_plain(context, folder, files)
|
||||
except UnauthorizedException:
|
||||
pass
|
||||
@@ -174,7 +180,7 @@ class LsCommand(BaseCommand):
|
||||
"updated_at": folder.updated_at,
|
||||
}
|
||||
if isinstance(folder, Folder):
|
||||
folder_api = FoldersApi(context.connection)
|
||||
folder_api = FoldersApi(context.client.connection)
|
||||
data["metadata"] = folder_api.metadata(folder.id)
|
||||
if context.is_recursive:
|
||||
sub_folders: list[dict[str, Any]] = []
|
||||
@@ -183,7 +189,7 @@ class LsCommand(BaseCommand):
|
||||
if sub_folder.lock:
|
||||
folder_api.auth(sub_folder.id, context.password)
|
||||
folder2 = folder_api.retrieve(sub_folder.id)
|
||||
files2 = cls._find_files(context.connection, sub_folder.id)
|
||||
files2 = context.client.find_files(sub_folder.id)
|
||||
sub_folders.append(cls._folder2dict(context, folder2, files2))
|
||||
except UnauthorizedException:
|
||||
pass
|
||||
@@ -205,7 +211,7 @@ class LsCommand(BaseCommand):
|
||||
# "thumbnail": file.thumbnail,
|
||||
"description": file.description,
|
||||
"metadata": file.metadata,
|
||||
"download_url": f"{context.connection.url}/{file.download_url}",
|
||||
"download_url": f"{context.client.connection.url}/{file.download_url}",
|
||||
"created_at": file.created_at,
|
||||
"updated_at": file.updated_at,
|
||||
}
|
||||
@@ -213,5 +219,5 @@ class LsCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def _laboratory_name(cls, context: LsCommandContext, laboratory_id: int) -> str:
|
||||
laboratory = context.connection.laboratories.find_by_id(laboratory_id)
|
||||
laboratory = context.client.connection.laboratories.find_by_id(laboratory_id)
|
||||
return laboratory.name if laboratory is not None else "(invalid)"
|
||||
|
||||
@@ -1,25 +1,7 @@
|
||||
import os
|
||||
from argparse import Namespace
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, MDRSException
|
||||
from mdrsclient.models import File, Folder
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UploadFileInfo:
|
||||
folder: Folder
|
||||
files: list[File]
|
||||
path: str
|
||||
|
||||
|
||||
class UploadCommand(BaseCommand):
|
||||
@@ -55,78 +37,3 @@ class UploadCommand(BaseCommand):
|
||||
client = MdrsClient.from_remote(remote)
|
||||
client.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def _upload_logic(
|
||||
cls, connection, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool
|
||||
) -> None:
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
l_path = os.path.abspath(local_path)
|
||||
if not os.path.exists(l_path):
|
||||
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
||||
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_path)
|
||||
files = cls._find_files(connection, folder.id)
|
||||
infos: list[UploadFileInfo] = []
|
||||
if os.path.isdir(l_path):
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
|
||||
folder_api = FoldersApi(connection)
|
||||
folder_map: dict[str, Folder] = {}
|
||||
folder_map[r_path] = folder
|
||||
files_map: dict[str, list[File]] = {}
|
||||
files_map[r_path] = files
|
||||
l_basename = os.path.basename(l_path)
|
||||
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
||||
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
||||
d_dirname = os.path.join(r_path, sub)
|
||||
d_basename = os.path.basename(d_dirname)
|
||||
# prepare destination parent path
|
||||
d_parent_dirname = os.path.dirname(d_dirname)
|
||||
if folder_map.get(d_parent_dirname) is None:
|
||||
parent_folder = cls._find_folder(connection, laboratory, d_parent_dirname)
|
||||
folder_map[d_parent_dirname] = parent_folder
|
||||
parent_files = cls._find_files(connection, parent_folder.id)
|
||||
files_map[d_parent_dirname] = parent_files
|
||||
# prepare destination path
|
||||
if folder_map.get(d_dirname) is None:
|
||||
d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
|
||||
if d_folder is None:
|
||||
d_folder_id = folder_api.create(normalize("NFC", d_basename), folder_map[d_parent_dirname].id)
|
||||
else:
|
||||
d_folder_id = d_folder.id
|
||||
print(d_dirname)
|
||||
folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
|
||||
files_map[d_dirname] = cls._find_files(connection, d_folder_id)
|
||||
if d_folder is None:
|
||||
folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
|
||||
# register upload file list
|
||||
for filename in filenames:
|
||||
infos.append(
|
||||
UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
|
||||
)
|
||||
else:
|
||||
infos.append(UploadFileInfo(folder, files, l_path))
|
||||
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
||||
|
||||
@classmethod
|
||||
def __multiple_upload(
|
||||
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
|
||||
) -> None:
|
||||
file_api = FilesApi(connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
||||
|
||||
@classmethod
|
||||
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||
basename = os.path.basename(info.path)
|
||||
file = find_file(info.files, basename)
|
||||
try:
|
||||
if file is None:
|
||||
file_api.create(info.folder.id, info.path)
|
||||
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
||||
file_api.update(file, info.path)
|
||||
print(os.path.join(info.folder.path, basename))
|
||||
except MDRSException as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
@@ -2,9 +2,6 @@ from argparse import Namespace
|
||||
from typing import Any, Final
|
||||
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import MissingConfigurationException
|
||||
|
||||
|
||||
class WhoamiCommand(BaseCommand):
|
||||
@@ -23,9 +20,9 @@ class WhoamiCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def whoami(cls, remote: str) -> None:
|
||||
remote_host = cls._parse_remote_host(remote)
|
||||
from mdrsclient.client import MdrsClient
|
||||
|
||||
remote_host = MdrsClient.parse_remote_host(remote)
|
||||
client = MdrsClient.from_remote(remote_host)
|
||||
if client.connection.token is not None and client.connection.token.is_expired:
|
||||
client.logout()
|
||||
|
||||
+52
-2
@@ -1,6 +1,7 @@
|
||||
import configparser
|
||||
import os
|
||||
from typing import Final
|
||||
import threading
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
import validators
|
||||
|
||||
@@ -9,7 +10,56 @@ from mdrsclient.settings import CONFIG_DIRNAME
|
||||
from mdrsclient.utils import FileLock
|
||||
|
||||
|
||||
class ConfigFile:
|
||||
@runtime_checkable
|
||||
class ConfigInterface(Protocol):
|
||||
remote: str
|
||||
|
||||
def list(self) -> list[tuple[str, str]]: ...
|
||||
@property
|
||||
def url(self) -> str | None: ...
|
||||
@url.setter
|
||||
def url(self, url: str) -> None: ...
|
||||
@url.deleter
|
||||
def url(self) -> None: ...
|
||||
|
||||
|
||||
class InMemoryConfig(ConfigInterface):
|
||||
__configs: dict[str, str] = {}
|
||||
__lock: threading.Lock = threading.Lock()
|
||||
remote: str
|
||||
|
||||
def __init__(self, remote: str) -> None:
|
||||
self.remote = remote
|
||||
|
||||
def list(self) -> list[tuple[str, str]]:
|
||||
with self.__lock:
|
||||
return list(self.__configs.items())
|
||||
|
||||
@property
|
||||
def url(self) -> str | None:
|
||||
with self.__lock:
|
||||
return self.__configs.get(self.remote)
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str) -> None:
|
||||
if not validators.url(url):
|
||||
raise IllegalArgumentException("malformed URI sequence")
|
||||
with self.__lock:
|
||||
self.__configs[self.remote] = url
|
||||
|
||||
@url.deleter
|
||||
def url(self) -> None:
|
||||
with self.__lock:
|
||||
if self.remote in self.__configs:
|
||||
del self.__configs[self.remote]
|
||||
|
||||
@classmethod
|
||||
def clear(cls) -> None:
|
||||
with cls.__lock:
|
||||
cls.__configs.clear()
|
||||
|
||||
|
||||
class ConfigFile(ConfigInterface):
|
||||
OPTION_URL: Final[str] = "url"
|
||||
CONFIG_FILENAME: Final[str] = "config.ini"
|
||||
remote: str
|
||||
|
||||
+16
-6
@@ -1,10 +1,11 @@
|
||||
from typing import Any
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from mdrsclient.api import DoiApi, FilesApi, FoldersApi, LaboratoriesApi, UsersApi
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.cache import CacheInterface
|
||||
from mdrsclient.config import ConfigFile, ConfigInterface
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import (
|
||||
IllegalArgumentException,
|
||||
@@ -17,15 +18,24 @@ from mdrsclient.utils import page_num_from_url
|
||||
|
||||
|
||||
class MdrsService:
|
||||
def __init__(self, connection: MDRSConnection):
|
||||
config_class: type[ConfigInterface] = ConfigFile
|
||||
|
||||
def __init__(self, connection: MDRSConnection, config_class: type[ConfigInterface] | None = None):
|
||||
self.connection = connection
|
||||
if config_class is not None:
|
||||
self.config_class = config_class
|
||||
|
||||
@classmethod
|
||||
def create_connection(cls, remote: str) -> MDRSConnection:
|
||||
config = ConfigFile(remote)
|
||||
def create_connection(
|
||||
cls, remote: str, cache: CacheInterface | None = None, config: ConfigInterface | None = None
|
||||
) -> MDRSConnection:
|
||||
if config is None:
|
||||
config = ConfigFile(remote)
|
||||
elif config.remote != remote:
|
||||
raise IllegalArgumentException("Remote host parameter mismatch.")
|
||||
if config.url is None:
|
||||
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
|
||||
return MDRSConnection(config.remote, config.url)
|
||||
return MDRSConnection(config.remote, config.url, cache=cache)
|
||||
|
||||
def login(self, username: str, password: str) -> tuple[Token, User]:
|
||||
user_api = UsersApi(self.connection)
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.exceptions import IllegalArgumentException, MDRSException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UploadFileInfo:
|
||||
folder: Folder
|
||||
files: list[File]
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadFileInfo:
|
||||
file: File
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
hasError: bool
|
||||
isSkipIfExists: bool
|
||||
files: list[DownloadFileInfo]
|
||||
|
||||
|
||||
class Uploader:
|
||||
def __init__(self, client: Any) -> None:
|
||||
self.client = client
|
||||
|
||||
def upload(
|
||||
self, local_path: str, remote_path: str, is_recursive: bool = False, is_skip_if_exists: bool = False
|
||||
) -> None:
|
||||
remote, laboratory_name, r_path = self.client.parse_remote_host_with_path(remote_path)
|
||||
l_path = os.path.abspath(local_path)
|
||||
if not os.path.exists(l_path):
|
||||
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
||||
|
||||
laboratory = self.client.find_laboratory(laboratory_name)
|
||||
folder = self.client.find_folder(laboratory, r_path)
|
||||
files = self.client.find_files(folder.id)
|
||||
infos: list[UploadFileInfo] = []
|
||||
if os.path.isdir(l_path):
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
|
||||
folder_api = FoldersApi(self.client.connection)
|
||||
folder_map: dict[str, Folder] = {}
|
||||
folder_map[r_path] = folder
|
||||
files_map: dict[str, list[File]] = {}
|
||||
files_map[r_path] = files
|
||||
l_basename = os.path.basename(l_path)
|
||||
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
||||
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
||||
d_dirname = os.path.join(r_path, sub)
|
||||
d_basename = os.path.basename(d_dirname)
|
||||
# prepare destination parent path
|
||||
d_parent_dirname = os.path.dirname(d_dirname)
|
||||
if folder_map.get(d_parent_dirname) is None:
|
||||
parent_folder = self.client.find_folder(laboratory, d_parent_dirname)
|
||||
folder_map[d_parent_dirname] = parent_folder
|
||||
parent_files = self.client.find_files(parent_folder.id)
|
||||
files_map[d_parent_dirname] = parent_files
|
||||
# prepare destination path
|
||||
if folder_map.get(d_dirname) is None:
|
||||
d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
|
||||
if d_folder is None:
|
||||
d_folder_id = folder_api.create(normalize("NFC", d_basename), folder_map[d_parent_dirname].id)
|
||||
else:
|
||||
d_folder_id = d_folder.id
|
||||
print(d_dirname)
|
||||
folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
|
||||
files_map[d_dirname] = self.client.find_files(d_folder_id)
|
||||
if d_folder is None:
|
||||
folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
|
||||
# register upload file list
|
||||
for filename in filenames:
|
||||
infos.append(
|
||||
UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
|
||||
)
|
||||
else:
|
||||
infos.append(UploadFileInfo(folder, files, l_path))
|
||||
self.__multiple_upload(infos, is_skip_if_exists)
|
||||
|
||||
def __multiple_upload(self, infos: list[UploadFileInfo], is_skip_if_exists: bool) -> None:
|
||||
file_api = FilesApi(self.client.connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
pool.map(lambda x: self.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
||||
|
||||
def __multiple_upload_worker(self, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||
basename = os.path.basename(info.path)
|
||||
file = find_file(info.files, basename)
|
||||
try:
|
||||
if file is None:
|
||||
file_api.create(info.folder.id, info.path)
|
||||
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
||||
file_api.update(file, info.path)
|
||||
print(os.path.join(info.folder.path, basename))
|
||||
except MDRSException as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(self, client: Any) -> None:
|
||||
self.client = client
|
||||
|
||||
def download(
|
||||
self,
|
||||
remote_path: str,
|
||||
local_path: str,
|
||||
is_recursive: bool = False,
|
||||
is_skip_if_exists: bool = False,
|
||||
password: str | None = None,
|
||||
excludes: list[str] | None = None,
|
||||
) -> None:
|
||||
excludes_clean = excludes or []
|
||||
# Detect DOI path: "remote:10.xxxx/prefix.ID[/optional/sub/path]"
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if self.client.is_doi(path_component):
|
||||
remote, doi, subpath = self.client.parse_doi_remote_host(remote_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
doi_folder, laboratory = self.client.find_folder_by_doi(doi, password)
|
||||
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
folder = doi_folder
|
||||
is_folder = True
|
||||
else:
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
r_parent_folder = self.client.find_folder(laboratory, abs_path, password)
|
||||
r_parent_files = self.client.find_files(r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if self.__check_excludes(excludes_clean, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
self.__multiple_download(context)
|
||||
return
|
||||
else:
|
||||
folder_simple = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder_simple is None:
|
||||
raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
|
||||
folder = FoldersApi(self.client.connection).retrieve(folder_simple.id)
|
||||
is_folder = True
|
||||
|
||||
# For a DOI target the whole folder is the download target.
|
||||
if not is_recursive:
|
||||
# Non-recursive: download only the files at the top level of the DOI folder.
|
||||
files = self.client.find_files(folder.id)
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
for file in files:
|
||||
if self.__check_excludes(excludes_clean, laboratory, folder, file):
|
||||
continue
|
||||
l_path = os.path.join(l_dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
self.__multiple_download(context)
|
||||
return
|
||||
folder_api = FoldersApi(self.client.connection)
|
||||
self.__multiple_download_pickup_recursive_files(
|
||||
folder_api, laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
|
||||
)
|
||||
return
|
||||
|
||||
remote, laboratory_name, r_path = self.client.parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
laboratory = self.client.find_laboratory(laboratory_name)
|
||||
r_parent_folder = self.client.find_folder(laboratory, r_dirname, password)
|
||||
r_parent_files = self.client.find_files(r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if self.__check_excludes(excludes_clean, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
self.__multiple_download(context)
|
||||
else:
|
||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder is None:
|
||||
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
folder_api = FoldersApi(self.client.connection)
|
||||
self.__multiple_download_pickup_recursive_files(
|
||||
folder_api, laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
|
||||
)
|
||||
|
||||
def __multiple_download_pickup_recursive_files(
|
||||
self,
|
||||
folder_api: FoldersApi,
|
||||
laboratory: Laboratory,
|
||||
folder_id: str,
|
||||
basedir: str,
|
||||
excludes: list[str],
|
||||
is_skip_if_exists: bool,
|
||||
) -> None:
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
folder = folder_api.retrieve(folder_id)
|
||||
files = self.client.find_files(folder.id)
|
||||
dirname = os.path.join(basedir, folder.name)
|
||||
if self.__check_excludes(excludes, laboratory, folder, None):
|
||||
return
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
print(dirname)
|
||||
for file in files:
|
||||
if self.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
path = os.path.join(dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, path))
|
||||
self.__multiple_download(context)
|
||||
if context.hasError:
|
||||
raise UnexpectedException("Some files failed to download.")
|
||||
for sub_folder in folder.sub_folders:
|
||||
self.__multiple_download_pickup_recursive_files(
|
||||
folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
|
||||
def __multiple_download(self, context: DownloadContext) -> None:
|
||||
file_api = FilesApi(self.client.connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
results = pool.map(
|
||||
lambda x: self.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
|
||||
)
|
||||
hasError = next(filter(lambda x: x is False, results), None)
|
||||
if hasError is not None:
|
||||
context.hasError = True
|
||||
|
||||
def __multiple_download_worker(self, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||
if not is_skip_if_exists or not os.path.exists(info.path) or info.file.size != os.path.getsize(info.path):
|
||||
try:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
print(f"Failed: {info.path}")
|
||||
if os.path.isfile(info.path):
|
||||
os.remove(info.path)
|
||||
return False
|
||||
print(info.path)
|
||||
return True
|
||||
|
||||
def __check_excludes(self, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
|
||||
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
|
||||
return path in excludes
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "mdrs-client-python"
|
||||
version = "1.3.17"
|
||||
version = "1.3.18"
|
||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||
license = "MIT"
|
||||
@@ -28,7 +28,7 @@ requests = "^2.34.2"
|
||||
requests-toolbelt = "^1.0.0"
|
||||
python-dotenv = "^1.1.0"
|
||||
pydantic = "^2.13.4"
|
||||
pydantic-settings = "^2.14.1"
|
||||
pydantic-settings = "^2.14.2"
|
||||
PyJWT = "^2.13.0"
|
||||
validators = "^0.35.0"
|
||||
|
||||
@@ -37,7 +37,7 @@ black = "^26.5.1"
|
||||
flake8 = "^7.2.0"
|
||||
Flake8-pyproject = "^1.2.3"
|
||||
isort = "^8.0.1"
|
||||
pyright = "^1.1.401"
|
||||
pyright = "^1.1.411"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
mdrs = 'mdrsclient.__main__:main'
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Mark tests directory as a Python package
|
||||
@@ -0,0 +1,418 @@
|
||||
import argparse
|
||||
import json
|
||||
import unittest
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from mdrsclient.client import MdrsClient
|
||||
from mdrsclient.commands import (
|
||||
ChaclCommand,
|
||||
ConfigCommand,
|
||||
CpCommand,
|
||||
DownloadCommand,
|
||||
FileMetadataCommand,
|
||||
LabsCommand,
|
||||
LoginCommand,
|
||||
LogoutCommand,
|
||||
LsCommand,
|
||||
MetadataCommand,
|
||||
MkdirCommand,
|
||||
MvCommand,
|
||||
RmCommand,
|
||||
UploadCommand,
|
||||
VersionCommand,
|
||||
WhoamiCommand,
|
||||
)
|
||||
from mdrsclient.models import Folder, Laboratory
|
||||
|
||||
|
||||
class TestCommands(unittest.TestCase):
|
||||
def parse_args(self, cmd_class, args_list):
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(title="subcommands")
|
||||
cmd_class.register(subparsers)
|
||||
return parser.parse_args(args_list)
|
||||
|
||||
def test_version_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.return_value = mock_client
|
||||
mock_client.version.return_value = "mdrs 1.3.17"
|
||||
|
||||
args = self.parse_args(VersionCommand, ["version"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertEqual(fake_out.getvalue().strip(), "mdrs 1.3.17")
|
||||
|
||||
mock_client_class.assert_called_once_with(None)
|
||||
mock_client.version.assert_called_once()
|
||||
|
||||
def test_login_command_with_args(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
|
||||
args = self.parse_args(LoginCommand, ["login", "-u", "myuser", "-p", "mypass", "myremote"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertIn("Login Successful", fake_out.getvalue())
|
||||
|
||||
mock_client_class.parse_remote_host.assert_called_once_with("myremote")
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.login.assert_called_once_with("myuser", "mypass")
|
||||
|
||||
def test_login_command_interactive(self):
|
||||
with (
|
||||
patch("mdrsclient.client.MdrsClient") as mock_client_class,
|
||||
patch("builtins.input", return_value="myuser_int") as mock_input,
|
||||
patch("getpass.getpass", return_value="mypass_int") as mock_getpass,
|
||||
):
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
|
||||
args = self.parse_args(LoginCommand, ["login", "myremote"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertIn("Login Successful", fake_out.getvalue())
|
||||
|
||||
mock_input.assert_called_once_with("Username: ")
|
||||
mock_getpass.assert_called_once_with("Password: ")
|
||||
mock_client.login.assert_called_once_with("myuser_int", "mypass_int")
|
||||
|
||||
def test_logout_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
|
||||
args = self.parse_args(LogoutCommand, ["logout", "myremote"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.parse_remote_host.assert_called_once_with("myremote")
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.logout.assert_called_once()
|
||||
|
||||
def test_whoami_command_logged_in(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
mock_client.connection.token = None
|
||||
|
||||
mock_user = MagicMock()
|
||||
mock_user.username = "test_user"
|
||||
mock_client.whoami.return_value = mock_user
|
||||
|
||||
args = self.parse_args(WhoamiCommand, ["whoami", "myremote"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertEqual(fake_out.getvalue().strip(), "test_user")
|
||||
|
||||
mock_client_class.parse_remote_host.assert_called_once_with("myremote")
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.whoami.assert_called_once()
|
||||
|
||||
def test_whoami_command_anonymous(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
mock_client.connection.token = None
|
||||
mock_client.whoami.side_effect = Exception("Not logged in")
|
||||
|
||||
args = self.parse_args(WhoamiCommand, ["whoami", "myremote"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertEqual(fake_out.getvalue().strip(), "(Anonymous)")
|
||||
|
||||
def test_labs_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client_class.parse_remote_host.return_value = "myremote"
|
||||
|
||||
mock_lab = MagicMock()
|
||||
mock_lab.id = 1
|
||||
mock_lab.name = "lab_name"
|
||||
mock_lab.pi_name = "pi_name"
|
||||
mock_lab.full_name = "full_name"
|
||||
mock_client.get_laboratories.return_value = [mock_lab]
|
||||
|
||||
args = self.parse_args(LabsCommand, ["labs", "myremote"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
output = fake_out.getvalue()
|
||||
self.assertIn("Name", output)
|
||||
self.assertIn("lab_name", output)
|
||||
self.assertIn("pi_name", output)
|
||||
|
||||
mock_client_class.parse_remote_host.assert_called_once_with("myremote")
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.get_laboratories.assert_called_once()
|
||||
|
||||
def test_ls_command_plain(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock(spec=MdrsClient)
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
mock_folder = Folder(
|
||||
id="folder_id",
|
||||
pid=None,
|
||||
name="root",
|
||||
access_level=1,
|
||||
lock=False,
|
||||
size=0,
|
||||
laboratory_id=1,
|
||||
description="",
|
||||
created_at="2026-07-02T00:00:00Z",
|
||||
updated_at="2026-07-02T00:00:00Z",
|
||||
restrict_opened_at=None,
|
||||
metadata=[],
|
||||
sub_folders=[],
|
||||
path="/root",
|
||||
)
|
||||
|
||||
mock_lab = Laboratory(id=1, name="mylab", pi_name="pi_name", full_name="full_name")
|
||||
|
||||
mock_client.resolve_folder.return_value = (mock_folder, mock_lab)
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "file.txt"
|
||||
mock_file.size = 100
|
||||
mock_file.updated_at_name = "2026-07-02"
|
||||
mock_client.find_files.return_value = [mock_file]
|
||||
|
||||
args = self.parse_args(LsCommand, ["ls", "myremote:/mylab/"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
output = fake_out.getvalue()
|
||||
self.assertIn("Type", output)
|
||||
self.assertIn("file.txt", output)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.resolve_folder.assert_called_once_with("myremote:/mylab/", None)
|
||||
mock_client.find_files.assert_called_once_with(mock_folder.id)
|
||||
|
||||
def test_ls_command_json(self):
|
||||
with (
|
||||
patch("mdrsclient.client.MdrsClient") as mock_client_class,
|
||||
patch("mdrsclient.commands.ls.FoldersApi") as mock_folders_api_class,
|
||||
):
|
||||
mock_folders_api = MagicMock()
|
||||
mock_folders_api_class.return_value = mock_folders_api
|
||||
mock_folders_api.metadata.return_value = {"folder_meta": "val"}
|
||||
|
||||
mock_client = MagicMock(spec=MdrsClient)
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client.connection = MagicMock()
|
||||
|
||||
mock_folder = Folder(
|
||||
id="folder_id",
|
||||
pid="parent_id",
|
||||
name="root",
|
||||
access_level=1,
|
||||
lock=False,
|
||||
size=0,
|
||||
laboratory_id=1,
|
||||
description="Root folder",
|
||||
created_at="2026-07-02T00:00:00Z",
|
||||
updated_at="2026-07-02T00:00:00Z",
|
||||
restrict_opened_at=None,
|
||||
metadata=[],
|
||||
sub_folders=[],
|
||||
path="/root",
|
||||
)
|
||||
|
||||
mock_lab = Laboratory(id=1, name="mylab", pi_name="pi_name", full_name="full_name")
|
||||
|
||||
mock_client.resolve_folder.return_value = (mock_folder, mock_lab)
|
||||
mock_client.connection.laboratories.find_by_id.return_value = mock_lab
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.id = "file_id"
|
||||
mock_file.name = "file.txt"
|
||||
mock_file.type = "text"
|
||||
mock_file.size = 100
|
||||
mock_file.description = "A file"
|
||||
mock_file.metadata = {}
|
||||
mock_file.download_url = "download/file"
|
||||
mock_file.created_at = "2026-07-02T00:00:00Z"
|
||||
mock_file.updated_at = "2026-07-02T00:00:00Z"
|
||||
mock_client.find_files.return_value = [mock_file]
|
||||
|
||||
args = self.parse_args(LsCommand, ["ls", "-J", "myremote:/mylab/"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
output = fake_out.getvalue()
|
||||
parsed_json = json.loads(output)
|
||||
self.assertEqual(parsed_json["name"], "root")
|
||||
self.assertEqual(parsed_json["files"][0]["name"], "file.txt")
|
||||
|
||||
def test_mkdir_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(MkdirCommand, ["mkdir", "myremote:/mylab/newfolder"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.mkdir.assert_called_once_with("myremote:/mylab/newfolder")
|
||||
|
||||
def test_rm_command_file(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(RmCommand, ["rm", "myremote:/mylab/file.txt"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.rm.assert_called_once_with("myremote:/mylab/file.txt", False)
|
||||
|
||||
def test_rm_command_recursive(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(RmCommand, ["rm", "-r", "myremote:/mylab/folder"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.rm.assert_called_once_with("myremote:/mylab/folder", True)
|
||||
|
||||
def test_cp_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(CpCommand, ["cp", "myremote:/mylab/src", "myremote:/mylab/dest"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.cp.assert_called_once_with("myremote:/mylab/src", "myremote:/mylab/dest", False)
|
||||
|
||||
def test_mv_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(MvCommand, ["mv", "myremote:/mylab/src", "myremote:/mylab/dest"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.mv.assert_called_once_with("myremote:/mylab/src", "myremote:/mylab/dest")
|
||||
|
||||
def test_chacl_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(ChaclCommand, ["chacl", "-r", "-p", "secret", "private", "myremote:/mylab/"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.chacl.assert_called_once_with("myremote:/mylab/", 1, True, "secret")
|
||||
|
||||
def test_metadata_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client.metadata.return_value = {"meta_key": "meta_val"}
|
||||
|
||||
args = self.parse_args(MetadataCommand, ["metadata", "-p", "secret", "myremote:/mylab/"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
parsed_json = json.loads(fake_out.getvalue())
|
||||
self.assertEqual(parsed_json["meta_key"], "meta_val")
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.metadata.assert_called_once_with("myremote:/mylab/", "secret")
|
||||
|
||||
def test_file_metadata_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
mock_client.file_metadata.return_value = {"file_meta": "val"}
|
||||
|
||||
args = self.parse_args(FileMetadataCommand, ["file-metadata", "-p", "secret", "myremote:/mylab/file.txt"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
parsed_json = json.loads(fake_out.getvalue())
|
||||
self.assertEqual(parsed_json["file_meta"], "val")
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.file_metadata.assert_called_once_with("myremote:/mylab/file.txt", "secret")
|
||||
|
||||
def test_upload_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(UploadCommand, ["upload", "-r", "-s", "local_file.txt", "myremote:/mylab/"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.upload.assert_called_once_with("local_file.txt", "myremote:/mylab/", True, True)
|
||||
|
||||
def test_download_command(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.from_remote.return_value = mock_client
|
||||
|
||||
args = self.parse_args(
|
||||
DownloadCommand, ["download", "-r", "-s", "-p", "pass", "-e", "ex1", "myremote:/mylab/", "local_dir"]
|
||||
)
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.from_remote.assert_called_once_with("myremote")
|
||||
mock_client.download.assert_called_once_with("myremote:/mylab/", "local_dir", True, True, "pass", ["ex1"])
|
||||
|
||||
def test_config_create(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
args = self.parse_args(ConfigCommand, ["config", "create", "myremote", "http://example.com"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.assert_called_once_with(None)
|
||||
mock_client.config_create.assert_called_once_with("myremote", "http://example.com")
|
||||
|
||||
def test_config_update(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
args = self.parse_args(ConfigCommand, ["config", "update", "myremote", "http://example.com"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.assert_called_once_with(None)
|
||||
mock_client.config_update.assert_called_once_with("myremote", "http://example.com")
|
||||
|
||||
def test_config_delete(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
args = self.parse_args(ConfigCommand, ["config", "delete", "myremote"])
|
||||
args.func(args)
|
||||
|
||||
mock_client_class.assert_called_once_with(None)
|
||||
mock_client.config_delete.assert_called_once_with("myremote")
|
||||
|
||||
def test_config_list(self):
|
||||
with patch("mdrsclient.client.MdrsClient") as mock_client_class:
|
||||
mock_client = MagicMock()
|
||||
mock_client_class.return_value = mock_client
|
||||
mock_client.config_list.return_value = [("remote1", "url1"), ("remote2", "url2")]
|
||||
|
||||
args = self.parse_args(ConfigCommand, ["config", "list"])
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
args.func(args)
|
||||
self.assertEqual(fake_out.getvalue(), "remote1:\turl1\nremote2:\turl2\n")
|
||||
|
||||
mock_client_class.assert_called_once_with(None)
|
||||
mock_client.config_list.assert_called_once()
|
||||
Reference in New Issue
Block a user