Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
809140dfbc
|
|||
|
5bdf837941
|
|||
|
04c0003a61
|
|||
|
428be1289c
|
|||
|
4283481695
|
|||
|
ddb4300d85
|
|||
|
68670a6588
|
|||
| 6d8fd0a598 | |||
| 634b5f1a13 | |||
| 95f22ea5f9 | |||
|
fffb686023
|
|||
|
893bc77128
|
|||
|
7b3f1f2d09
|
|||
|
8a5e1b68b7
|
+5
-1
@@ -16,7 +16,11 @@
|
||||
"mdrsclient",
|
||||
"neurodata",
|
||||
"Neuroinformatics",
|
||||
"orcid",
|
||||
"RIKEN"
|
||||
],
|
||||
"ignorePaths": [".env", "__pycache__"]
|
||||
"ignorePaths": [
|
||||
".env",
|
||||
"__pycache__"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,6 +18,32 @@ Create remote host configuration
|
||||
mdrs config create neurodata https://neurodata.riken.jp/api
|
||||
```
|
||||
|
||||
### config update
|
||||
|
||||
Update the URL of a registered remote host.
|
||||
|
||||
```shell
|
||||
mdrs config update neurodata https://neurodata.riken.jp/api
|
||||
```
|
||||
|
||||
### config list
|
||||
|
||||
List registered remote hosts.
|
||||
|
||||
```shell
|
||||
mdrs config list
|
||||
mdrs config ls
|
||||
```
|
||||
|
||||
### config delete
|
||||
|
||||
Remove a registered remote host.
|
||||
|
||||
```shell
|
||||
mdrs config delete neurodata
|
||||
mdrs config rm neurodata
|
||||
```
|
||||
|
||||
### login
|
||||
|
||||
Login to remote host
|
||||
@@ -56,13 +82,17 @@ mdrs labs neurodata:
|
||||
|
||||
### ls
|
||||
|
||||
List the folder contents
|
||||
List the folder contents. You can also specify a DOI path in the form `remote:10.xxxx/yyy.ID[/optional/subpath]`.
|
||||
|
||||
```shell
|
||||
mdrs ls neurodata:/NIU/Repository/
|
||||
mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
||||
mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
||||
|
||||
# DOI access examples:
|
||||
mdrs ls neurodata:10.60178/cbs.20260429-001
|
||||
mdrs ls "neurodata:10.60178/cbs.20260429-001/Figure 1"
|
||||
```
|
||||
|
||||
### mkdir
|
||||
@@ -85,7 +115,7 @@ mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
|
||||
|
||||
### download
|
||||
|
||||
Download the file or folder
|
||||
Download the file or folder. You can also specify a DOI path.
|
||||
|
||||
```shell
|
||||
mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
||||
@@ -93,6 +123,10 @@ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
|
||||
# DOI access examples:
|
||||
mdrs download neurodata:10.60178/cbs.20260429-001/README_NeuroData.md ./
|
||||
mdrs download -r "neurodata:10.60178/cbs.20260429-001/Figure 1" ./
|
||||
```
|
||||
|
||||
### mv
|
||||
@@ -134,20 +168,34 @@ mdrs chacl pw_open -r -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||
|
||||
### metadata
|
||||
|
||||
Get a folder metadata
|
||||
Get a folder metadata. You can also specify a DOI path.
|
||||
|
||||
```shell
|
||||
mdrs metadata neurodata:/NIU/Repository/TEST/
|
||||
mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
|
||||
# DOI access examples:
|
||||
mdrs metadata neurodata:10.60178/cbs.20260429-001
|
||||
```
|
||||
|
||||
### file-metadata
|
||||
|
||||
Get the file metadata
|
||||
Get the file metadata. You can also specify a DOI path.
|
||||
|
||||
```shell
|
||||
mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
||||
mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||
|
||||
# DOI access examples:
|
||||
mdrs file-metadata "neurodata:10.60178/cbs.20260429-001/Figure 1/Figure1v3.pdf"
|
||||
```
|
||||
|
||||
### version
|
||||
|
||||
Show the tool name and version number
|
||||
|
||||
```shell
|
||||
mdrs version
|
||||
```
|
||||
|
||||
### help
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
1.3.10
|
||||
@@ -17,6 +17,7 @@ from mdrsclient.commands import (
|
||||
MvCommand,
|
||||
RmCommand,
|
||||
UploadCommand,
|
||||
VersionCommand,
|
||||
WhoamiCommand,
|
||||
)
|
||||
from mdrsclient.exceptions import MDRSException
|
||||
@@ -29,6 +30,7 @@ def main() -> None:
|
||||
parsers = parser.add_subparsers(title="subcommands")
|
||||
|
||||
ConfigCommand.register(parsers)
|
||||
VersionCommand.register(parsers)
|
||||
LoginCommand.register(parsers)
|
||||
LogoutCommand.register(parsers)
|
||||
WhoamiCommand.register(parsers)
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import os
|
||||
from importlib.metadata import version
|
||||
|
||||
here = os.path.realpath(os.path.dirname(__file__))
|
||||
|
||||
with open(os.path.join(here, "VERSION")) as version_file:
|
||||
__version__ = version_file.read().strip()
|
||||
__version__ = version("mdrs-client-python")
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
from mdrsclient.api.doi import DoiApi
|
||||
from mdrsclient.api.files import FilesApi
|
||||
from mdrsclient.api.folders import FoldersApi
|
||||
from mdrsclient.api.laboratories import LaboratoriesApi
|
||||
from mdrsclient.api.users import UsersApi
|
||||
|
||||
__all__ = [
|
||||
"DoiApi",
|
||||
"FilesApi",
|
||||
"FoldersApi",
|
||||
"LaboratoriesApi",
|
||||
|
||||
@@ -5,12 +5,7 @@ from pydantic import TypeAdapter
|
||||
from requests import Response
|
||||
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import (
|
||||
BadRequestException,
|
||||
ForbiddenException,
|
||||
UnauthorizedException,
|
||||
UnexpectedException,
|
||||
)
|
||||
from mdrsclient.exceptions import BadRequestException, ForbiddenException, UnauthorizedException, UnexpectedException
|
||||
from mdrsclient.models.error import DRFStandardizedErrors
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api.base import BaseApi
|
||||
from mdrsclient.api.utils import token_check
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DoiFolderRef:
|
||||
"""Nested folder reference returned inside a DOI response.
|
||||
|
||||
The DOI endpoint only returns the folder ``id``; ``laboratory_id`` must be
|
||||
obtained by subsequently calling the folder retrieve endpoint.
|
||||
"""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DoiResponse:
|
||||
"""Response from GET v3/doi/{id}/."""
|
||||
|
||||
# The internal DOI suffix ID returned as a string (e.g. "20260429-001").
|
||||
id: str
|
||||
doi: str
|
||||
folder: DoiFolderRef
|
||||
|
||||
|
||||
class DoiApi(BaseApi):
|
||||
ENTRYPOINT: Final[str] = "v3/doi/"
|
||||
|
||||
def retrieve(self, doi_id: str) -> DoiResponse:
|
||||
"""Retrieve the folder associated with a DOI suffix ID (GET v3/doi/{id}/)."""
|
||||
url = self.ENTRYPOINT + doi_id + "/"
|
||||
token_check(self.connection)
|
||||
response = self.connection.get(url)
|
||||
self._raise_response_error(response)
|
||||
return TypeAdapter(DoiResponse).validate_python(response.json())
|
||||
+24
-2
@@ -1,6 +1,7 @@
|
||||
import mimetypes
|
||||
import os
|
||||
from typing import Any, Final
|
||||
from unicodedata import normalize
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic.dataclasses import dataclass
|
||||
@@ -17,10 +18,26 @@ class FilesApiCreateResponse:
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FilesApiListResponse:
|
||||
count: int
|
||||
next: str | None
|
||||
previous: str | None
|
||||
results: list[File]
|
||||
|
||||
|
||||
class FilesApi(BaseApi):
|
||||
ENTRYPOINT: Final[str] = "v3/files/"
|
||||
FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
|
||||
|
||||
def list(self, folder_id: str, page_num: int) -> FilesApiListResponse:
|
||||
url = self.ENTRYPOINT
|
||||
token_check(self.connection)
|
||||
params: dict[str, str | int] = {"folder_id": folder_id, "page": page_num}
|
||||
response = self.connection.get(url, params=params)
|
||||
self._raise_response_error(response)
|
||||
return TypeAdapter(FilesApiListResponse).validate_python(response.json())
|
||||
|
||||
def retrieve(self, id: str) -> File:
|
||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||
url = self.ENTRYPOINT + id + "/"
|
||||
@@ -37,7 +54,10 @@ class FilesApi(BaseApi):
|
||||
try:
|
||||
with open(os.path.realpath(path), mode="rb") as fp:
|
||||
data = MultipartEncoder(
|
||||
fields={"folder_id": folder_id, "file": (os.path.basename(path), fp, self._get_mime_type(path))}
|
||||
fields={
|
||||
"folder_id": folder_id,
|
||||
"file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path)),
|
||||
}
|
||||
)
|
||||
response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
|
||||
self._raise_response_error(response)
|
||||
@@ -59,7 +79,9 @@ class FilesApi(BaseApi):
|
||||
# update file body
|
||||
try:
|
||||
with open(os.path.realpath(path), mode="rb") as fp:
|
||||
data = MultipartEncoder(fields={"file": (os.path.basename(path), fp, self._get_mime_type(path))})
|
||||
data = MultipartEncoder(
|
||||
fields={"file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path))}
|
||||
)
|
||||
response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
|
||||
except OSError:
|
||||
raise UnexpectedException(f"Could not open `{path}` file.")
|
||||
|
||||
@@ -20,8 +20,10 @@ class UsersCurrentResponseLaboratory:
|
||||
class UsersApiCurrentResponse:
|
||||
id: int
|
||||
username: str
|
||||
full_name: str
|
||||
first_name: str
|
||||
last_name: str
|
||||
email: str
|
||||
orcid_id: str
|
||||
laboratories: list[UsersCurrentResponseLaboratory]
|
||||
is_staff: bool
|
||||
is_active: bool
|
||||
|
||||
@@ -12,6 +12,7 @@ from mdrsclient.commands.mkdir import MkdirCommand
|
||||
from mdrsclient.commands.mv import MvCommand
|
||||
from mdrsclient.commands.rm import RmCommand
|
||||
from mdrsclient.commands.upload import UploadCommand
|
||||
from mdrsclient.commands.version import VersionCommand
|
||||
from mdrsclient.commands.whoami import WhoamiCommand
|
||||
|
||||
__all__ = [
|
||||
@@ -29,5 +30,6 @@ __all__ = [
|
||||
"MvCommand",
|
||||
"RmCommand",
|
||||
"UploadCommand",
|
||||
"VersionCommand",
|
||||
"WhoamiCommand",
|
||||
]
|
||||
|
||||
+186
-2
@@ -1,9 +1,10 @@
|
||||
import os
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from mdrsclient.api import FoldersApi, LaboratoriesApi
|
||||
from mdrsclient.api import DoiApi, FilesApi, FoldersApi, LaboratoriesApi
|
||||
from mdrsclient.config import ConfigFile
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import (
|
||||
@@ -12,7 +13,8 @@ from mdrsclient.exceptions import (
|
||||
UnauthorizedException,
|
||||
UnexpectedException,
|
||||
)
|
||||
from mdrsclient.models import Folder, Laboratory
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.utils import page_num_from_url
|
||||
|
||||
|
||||
class BaseCommand(ABC):
|
||||
@@ -52,6 +54,19 @@ class BaseCommand(ABC):
|
||||
folder_api.auth(folders[0].id, password)
|
||||
return folder_api.retrieve(folders[0].id)
|
||||
|
||||
@classmethod
|
||||
def _find_files(cls, connection: MDRSConnection, folder_id: str) -> list[File]:
|
||||
files_api = FilesApi(connection)
|
||||
page = 1
|
||||
results_file = []
|
||||
while page:
|
||||
result = files_api.list(folder_id, page)
|
||||
results_file.extend(result.results)
|
||||
page = 0
|
||||
if result.next:
|
||||
page = page_num_from_url(result.next)
|
||||
return results_file
|
||||
|
||||
@classmethod
|
||||
def _parse_remote_host(cls, path: str) -> str:
|
||||
path_array = path.split(":")
|
||||
@@ -81,3 +96,172 @@ class BaseCommand(ABC):
|
||||
laboratory = folder_array.pop(0)
|
||||
folder = "/" + "/".join(folder_array)
|
||||
return (remote_host, laboratory, folder)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# DOI helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _is_doi(path_component: str) -> bool:
|
||||
"""Return True if path_component looks like a DOI string.
|
||||
|
||||
A DOI is recognised as a string that starts with ``10.`` and
|
||||
contains a ``/``.
|
||||
"""
|
||||
return path_component.startswith("10.") and "/" in path_component
|
||||
|
||||
@staticmethod
|
||||
def _doi_suffix_id(doi: str) -> str:
|
||||
"""Extract the internal system ID from a full DOI string.
|
||||
|
||||
MDRS uses the segment after the last ``.`` in the suffix (the part
|
||||
after the ``/``) as its identifier.
|
||||
Example: ``10.xxxx/prefix.20230511-001`` → ``20230511-001``.
|
||||
If there is no ``.`` in the suffix, the whole suffix is returned.
|
||||
Trailing slashes are stripped before processing.
|
||||
"""
|
||||
# Strip any trailing slash first.
|
||||
doi = doi.rstrip("/")
|
||||
slash_pos = doi.find("/")
|
||||
if slash_pos == -1:
|
||||
return doi
|
||||
suffix = doi[slash_pos + 1 :]
|
||||
dot_pos = suffix.rfind(".")
|
||||
return suffix[dot_pos + 1 :] if dot_pos != -1 else suffix
|
||||
|
||||
@staticmethod
|
||||
def _split_doi_and_subpath(doi_with_path: str) -> tuple[str, str]:
|
||||
"""Split a DOI-with-optional-path string into (doi, subpath)."""
|
||||
# Find the first '/' that separates registrant from suffix.
|
||||
first_slash = doi_with_path.find("/")
|
||||
if first_slash != -1:
|
||||
after_suffix_start = first_slash + 1
|
||||
after_first = doi_with_path[after_suffix_start:]
|
||||
# Find the next '/' inside the suffix portion — this starts the subpath.
|
||||
second_slash = after_first.find("/")
|
||||
if second_slash != -1:
|
||||
doi_end = after_suffix_start + second_slash
|
||||
doi = doi_with_path[:doi_end]
|
||||
subpath = doi_with_path[doi_end:] # begins with "/"
|
||||
# Treat a bare trailing slash as no subpath (root of DOI folder).
|
||||
if subpath == "/":
|
||||
return (doi, "")
|
||||
else:
|
||||
return (doi, subpath)
|
||||
else:
|
||||
# No second slash — the whole string is the DOI, no subpath.
|
||||
return (doi_with_path, "")
|
||||
else:
|
||||
return (doi_with_path, "")
|
||||
|
||||
@classmethod
|
||||
def _parse_doi_remote_host(cls, path: str) -> tuple[str, str, str]:
|
||||
"""Parse ``remote:10.xxxx/prefix.ID[/optional/sub/path]`` into ``(remote, doi, subpath)``."""
|
||||
parts = path.split(":", 1)
|
||||
if len(parts) != 2:
|
||||
raise IllegalArgumentException("remote_path must be in the form 'remote:10.xxxx/prefix.ID'")
|
||||
remote, doi_with_path = parts
|
||||
if not cls._is_doi(doi_with_path):
|
||||
raise IllegalArgumentException(
|
||||
f"Path `{doi_with_path}` does not look like a DOI (must start with '10.' and contain '/')."
|
||||
)
|
||||
doi, subpath = cls._split_doi_and_subpath(doi_with_path)
|
||||
return (remote, doi, subpath)
|
||||
|
||||
@classmethod
|
||||
def _find_folder_by_doi(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
doi: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory]:
|
||||
"""Resolve a DOI to a (Folder, Laboratory) pair.
|
||||
|
||||
Calls GET v3/doi/{id}/ to look up the folder ID, retrieves the full
|
||||
folder detail (which carries ``laboratory_id``), and resolves the
|
||||
laboratory from that field.
|
||||
"""
|
||||
doi_clean = doi.rstrip("/")
|
||||
doi_id = cls._doi_suffix_id(doi_clean)
|
||||
doi_api = DoiApi(connection)
|
||||
doi_resp = doi_api.retrieve(doi_id)
|
||||
|
||||
# Verify the returned DOI matches the one supplied (case-insensitive).
|
||||
returned_doi = doi_resp.doi.rstrip("/")
|
||||
if returned_doi.lower() != doi_clean.lower():
|
||||
raise IllegalArgumentException(
|
||||
f"DOI mismatch: requested `{doi_clean}` but server returned `{returned_doi}`."
|
||||
)
|
||||
|
||||
folder_api = FoldersApi(connection)
|
||||
|
||||
# Retrieve full folder detail directly by ID; laboratory_id is here.
|
||||
folder = folder_api.retrieve(doi_resp.folder.id)
|
||||
|
||||
if folder.lock:
|
||||
if password is None:
|
||||
raise UnauthorizedException(f"Folder for DOI `{doi_clean}` is locked.")
|
||||
folder_api.auth(doi_resp.folder.id, password)
|
||||
|
||||
# Resolve laboratory using laboratory_id from the full folder detail.
|
||||
lab_api = LaboratoriesApi(connection)
|
||||
labs = lab_api.list()
|
||||
lab = labs.find_by_id(folder.laboratory_id)
|
||||
if lab is None:
|
||||
raise UnexpectedException(f"Laboratory with id {folder.laboratory_id} not found.")
|
||||
|
||||
connection.laboratories = labs
|
||||
return (folder, lab)
|
||||
|
||||
@classmethod
|
||||
def _resolve_folder(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
remote_path: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory]:
|
||||
"""Resolve any remote path (normal or DOI) into a (Folder, Laboratory) pair."""
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
if not subpath:
|
||||
return (doi_folder, laboratory)
|
||||
else:
|
||||
abs_path = doi_folder.path.rstrip("/") + subpath
|
||||
folder = cls._find_folder(connection, laboratory, abs_path, password)
|
||||
return (folder, laboratory)
|
||||
else:
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_path, password)
|
||||
return (folder, laboratory)
|
||||
|
||||
@classmethod
|
||||
def _resolve_file(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
remote_path: str,
|
||||
password: str | None = None,
|
||||
) -> tuple[Folder, Laboratory, str]:
|
||||
"""Resolve a remote path pointing to a file into the parent Folder, its Laboratory, and the file's basename."""
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
raise IllegalArgumentException("DOI path must point to a file, not a folder.")
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
parent_folder = cls._find_folder(connection, laboratory, abs_path, password)
|
||||
return (parent_folder, laboratory, r_basename)
|
||||
else:
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
return (parent_folder, laboratory, r_basename)
|
||||
|
||||
@@ -31,7 +31,7 @@ class ChaclCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def chacl(cls, remote_path: str, access_level: int, is_recursive: bool, password: str | None) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
|
||||
@@ -26,10 +26,11 @@ class ConfigCommand(BaseCommand):
|
||||
update_parser.set_defaults(func=cls.func_update)
|
||||
# config list
|
||||
list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
|
||||
list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
|
||||
list_parser.set_defaults(func=cls.func_list)
|
||||
# config delete
|
||||
delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host", aliases=["remove"])
|
||||
delete_parser = config_parsers.add_parser(
|
||||
"delete", help="delete an existing remote host", aliases=["remove", "rm"]
|
||||
)
|
||||
delete_parser.add_argument("remote", help="label of remote host")
|
||||
delete_parser.set_defaults(func=cls.func_delete)
|
||||
|
||||
@@ -47,8 +48,7 @@ class ConfigCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def func_list(cls, args: Namespace) -> None:
|
||||
is_long = bool(args.long)
|
||||
cls.list(is_long)
|
||||
cls.list()
|
||||
|
||||
@classmethod
|
||||
def func_delete(cls, args: Namespace) -> None:
|
||||
@@ -74,13 +74,10 @@ class ConfigCommand(BaseCommand):
|
||||
config.url = url
|
||||
|
||||
@classmethod
|
||||
def list(cls, is_long: bool) -> None:
|
||||
def list(cls) -> None:
|
||||
config = ConfigFile("")
|
||||
for remote, url in config.list():
|
||||
line = f"{remote}:"
|
||||
if is_long:
|
||||
line += f"\t{url}"
|
||||
print(line)
|
||||
print(f"{remote}:\t{url}")
|
||||
|
||||
@classmethod
|
||||
def delete(cls, remote: str) -> None:
|
||||
|
||||
@@ -6,6 +6,7 @@ from unicodedata import normalize
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models.file import find_file
|
||||
|
||||
|
||||
class CpCommand(BaseCommand):
|
||||
@@ -28,8 +29,8 @@ class CpCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def cp(cls, src_path: str, dest_path: str, is_recursive: bool) -> None:
|
||||
(s_remote, s_laboratory_name, s_path) = cls._parse_remote_host_with_path(src_path)
|
||||
(d_remote, d_laboratory_name, d_path) = cls._parse_remote_host_with_path(dest_path)
|
||||
s_remote, s_laboratory_name, s_path = cls._parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = cls._parse_remote_host_with_path(dest_path)
|
||||
if s_remote != d_remote:
|
||||
raise IllegalArgumentException("Remote host mismatched.")
|
||||
if s_laboratory_name != d_laboratory_name:
|
||||
@@ -46,11 +47,13 @@ class CpCommand(BaseCommand):
|
||||
connection = cls._create_connection(s_remote)
|
||||
laboratory = cls._find_laboratory(connection, s_laboratory_name)
|
||||
s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
|
||||
s_parent_files = cls._find_files(connection, s_parent_folder.id)
|
||||
d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
|
||||
s_file = s_parent_folder.find_file(s_basename)
|
||||
d_parent_files = cls._find_files(connection, d_parent_folder.id)
|
||||
s_file = find_file(s_parent_files, s_basename)
|
||||
if s_file is not None:
|
||||
# source is file
|
||||
d_file = d_parent_folder.find_file(d_basename)
|
||||
d_file = find_file(d_parent_files, d_basename)
|
||||
if d_file is not None:
|
||||
raise IllegalArgumentException(f"File `{d_basename}` already exists.")
|
||||
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||
@@ -66,7 +69,7 @@ class CpCommand(BaseCommand):
|
||||
# source is folder
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot copy `{s_path}`: Is a folder.")
|
||||
if d_parent_folder.find_file(d_basename) is not None:
|
||||
if find_file(d_parent_files, d_basename) is not None:
|
||||
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
|
||||
d_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||
if d_folder is not None:
|
||||
|
||||
@@ -10,6 +10,7 @@ from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@@ -67,7 +68,61 @@ class DownloadCommand(BaseCommand):
|
||||
password: str | None,
|
||||
excludes: list[str],
|
||||
) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
# Detect DOI path: "remote:10.xxxx/prefix.ID[/optional/sub/path]"
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if cls._is_doi(path_component):
|
||||
remote, doi, subpath = cls._parse_doi_remote_host(remote_path)
|
||||
connection = cls._create_connection(remote)
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
doi_folder, laboratory = cls._find_folder_by_doi(connection, doi, password)
|
||||
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
folder = doi_folder
|
||||
is_folder = True
|
||||
else:
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_parent_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
r_parent_folder = cls._find_folder(connection, laboratory, abs_parent_path, password)
|
||||
r_parent_files = cls._find_files(connection, r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
return
|
||||
else:
|
||||
folder_simple = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder_simple is None:
|
||||
raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
|
||||
folder = FoldersApi(connection).retrieve(folder_simple.id)
|
||||
is_folder = True
|
||||
|
||||
# For a DOI target the whole folder is the download target.
|
||||
if not is_recursive:
|
||||
# Non-recursive: download only the files at the top level of the DOI folder.
|
||||
files = cls._find_files(connection, folder.id)
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
for file in files:
|
||||
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
l_path = os.path.join(l_dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
return
|
||||
folder_api = FoldersApi(connection)
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
return
|
||||
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
@@ -77,7 +132,8 @@ class DownloadCommand(BaseCommand):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
file = r_parent_folder.find_file(r_basename)
|
||||
r_parent_files = cls._find_files(connection, r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
@@ -109,13 +165,14 @@ class DownloadCommand(BaseCommand):
|
||||
) -> None:
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
folder = folder_api.retrieve(folder_id)
|
||||
files = cls._find_files(connection, folder.id)
|
||||
dirname = os.path.join(basedir, folder.name)
|
||||
if cls.__check_excludes(excludes, laboratory, folder, None):
|
||||
return
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
print(dirname)
|
||||
for file in folder.files:
|
||||
for file in files:
|
||||
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
path = os.path.join(dirname, file.name)
|
||||
@@ -141,7 +198,7 @@ class DownloadCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||
if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
|
||||
if not is_skip_if_exists or not os.path.exists(info.path) or info.file.size != os.path.getsize(info.path):
|
||||
try:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
from mdrsclient.api import FilesApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models.file import find_file
|
||||
|
||||
|
||||
class FileMetadataCommand(BaseCommand):
|
||||
@@ -24,14 +25,11 @@ class FileMetadataCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def file_metadata(cls, remote_path: str, password: str | None) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
file = folder.find_file(r_basename)
|
||||
folder, laboratory, r_basename = cls._resolve_file(connection, remote_path, password)
|
||||
files = cls._find_files(connection, folder.id)
|
||||
file = find_file(files, r_basename)
|
||||
if file is None:
|
||||
raise IllegalArgumentException(f"File `{r_basename}` not found.")
|
||||
file_api = FilesApi(connection)
|
||||
|
||||
+30
-25
@@ -4,7 +4,7 @@ from typing import Any
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.api import FoldersApi
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import UnauthorizedException
|
||||
@@ -23,7 +23,7 @@ class LsCommandContext:
|
||||
laboratory: Laboratory
|
||||
password: str
|
||||
is_json: bool
|
||||
is_quick: bool
|
||||
is_quiet: bool
|
||||
is_recursive: bool
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ class LsCommand(BaseCommand):
|
||||
ls_parser.add_argument("-J", "--json", help="turn on json output", action="store_true")
|
||||
ls_parser.add_argument(
|
||||
"-q",
|
||||
"--quick",
|
||||
"--quiet",
|
||||
help="don't output header row. this option is forced if the -r option is specified",
|
||||
action="store_true",
|
||||
)
|
||||
@@ -49,35 +49,36 @@ class LsCommand(BaseCommand):
|
||||
password = str(args.password) if args.password else None
|
||||
is_json = bool(args.json)
|
||||
is_recursive = bool(args.recursive)
|
||||
is_quick = bool(args.quick) if not is_recursive else True
|
||||
cls.ls(remote_path, password, is_json, is_recursive, is_quick)
|
||||
is_quiet = bool(args.quiet) if not is_recursive else True
|
||||
cls.ls(remote_path, password, is_json, is_recursive, is_quiet)
|
||||
|
||||
@classmethod
|
||||
def ls(cls, remote_path: str, password: str | None, is_json: bool, is_recursive: bool, is_quick: bool) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
def ls(cls, remote_path: str, password: str | None, is_json: bool, is_recursive: bool, is_quiet: bool) -> None:
|
||||
remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder, laboratory = cls._resolve_folder(connection, remote_path, password)
|
||||
laboratory_name = laboratory.name
|
||||
files = cls._find_files(connection, folder.id)
|
||||
context = LsCommandContext(
|
||||
f"{remote}:/{laboratory_name}",
|
||||
connection,
|
||||
laboratory,
|
||||
password if password is not None else "",
|
||||
is_json,
|
||||
is_quick,
|
||||
is_quiet,
|
||||
is_recursive,
|
||||
)
|
||||
folder = cls._find_folder(connection, laboratory, r_path, password)
|
||||
if context.is_json:
|
||||
cls._ls_json(context, folder)
|
||||
cls._ls_json(context, folder, files)
|
||||
else:
|
||||
cls._ls_plain(context, folder)
|
||||
cls._ls_plain(context, folder, files)
|
||||
|
||||
@classmethod
|
||||
def _ls_json(cls, context: LsCommandContext, folder: Folder) -> None:
|
||||
print(json.dumps(cls._folder2dict(context, folder), ensure_ascii=False))
|
||||
def _ls_json(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
|
||||
print(json.dumps(cls._folder2dict(context, folder, files), ensure_ascii=False))
|
||||
|
||||
@classmethod
|
||||
def _ls_plain(cls, context: LsCommandContext, folder: Folder) -> None:
|
||||
def _ls_plain(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
|
||||
label = {
|
||||
"type": "Type",
|
||||
"acl": "Access",
|
||||
@@ -88,7 +89,7 @@ class LsCommand(BaseCommand):
|
||||
}
|
||||
length: dict[str, int] = {}
|
||||
for key in label.keys():
|
||||
length[key] = len(label[key]) if not context.is_quick else 0
|
||||
length[key] = len(label[key]) if not context.is_quiet else 0
|
||||
for sub_folder in folder.sub_folders:
|
||||
sub_laboratory = context.connection.laboratories.find_by_id(sub_folder.laboratory_id)
|
||||
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
|
||||
@@ -97,7 +98,7 @@ class LsCommand(BaseCommand):
|
||||
length["size"] = max(length["size"], len(str(folder.size)))
|
||||
length["date"] = max(length["date"], len(sub_folder.updated_at_name))
|
||||
length["name"] = max(length["name"], len(sub_folder.name))
|
||||
for file in folder.files:
|
||||
for file in files:
|
||||
length["size"] = max(length["size"], len(str(file.size)))
|
||||
length["date"] = max(length["date"], len(file.updated_at_name))
|
||||
length["name"] = max(length["name"], len(file.name))
|
||||
@@ -111,9 +112,9 @@ class LsCommand(BaseCommand):
|
||||
|
||||
if context.is_recursive:
|
||||
print(f"{context.prefix}{folder.path}:")
|
||||
print(f"total {sum(f.size for f in folder.files)}")
|
||||
print(f"total {sum(f.size for f in files)}")
|
||||
|
||||
if not context.is_quick:
|
||||
if not context.is_quiet:
|
||||
print(header)
|
||||
print("-" * len(header.expandtabs()))
|
||||
|
||||
@@ -125,7 +126,7 @@ class LsCommand(BaseCommand):
|
||||
f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.size:{length['size']}}\t"
|
||||
f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
|
||||
)
|
||||
for file in sorted(folder.files, key=lambda x: x.name):
|
||||
for file in sorted(files, key=lambda x: x.name):
|
||||
print(
|
||||
f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t"
|
||||
f"{context.laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t"
|
||||
@@ -140,12 +141,15 @@ class LsCommand(BaseCommand):
|
||||
if sub_folder.lock:
|
||||
folder_api.auth(sub_folder.id, context.password)
|
||||
folder = folder_api.retrieve(sub_folder.id)
|
||||
cls._ls_plain(context, folder)
|
||||
files = cls._find_files(context.connection, sub_folder.id)
|
||||
cls._ls_plain(context, folder, files)
|
||||
except UnauthorizedException:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def _folder2dict(cls, context: LsCommandContext, folder: Folder | FolderSimple) -> dict[str, Any]:
|
||||
def _folder2dict(
|
||||
cls, context: LsCommandContext, folder: Folder | FolderSimple, files: list[File]
|
||||
) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {
|
||||
"id": folder.id,
|
||||
"pid": folder.pid,
|
||||
@@ -168,15 +172,16 @@ class LsCommand(BaseCommand):
|
||||
if sub_folder.lock:
|
||||
folder_api.auth(sub_folder.id, context.password)
|
||||
folder2 = folder_api.retrieve(sub_folder.id)
|
||||
sub_folders.append(cls._folder2dict(context, folder2))
|
||||
files2 = cls._find_files(context.connection, sub_folder.id)
|
||||
sub_folders.append(cls._folder2dict(context, folder2, files2))
|
||||
except UnauthorizedException:
|
||||
pass
|
||||
data["sub_folders"] = sub_folders
|
||||
else:
|
||||
data["sub_folders"] = list(
|
||||
map(lambda x: cls._folder2dict(context, x), sorted(folder.sub_folders, key=lambda x: x.name))
|
||||
map(lambda x: cls._folder2dict(context, x, []), sorted(folder.sub_folders, key=lambda x: x.name))
|
||||
)
|
||||
data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(folder.files, key=lambda x: x.name)))
|
||||
data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(files, key=lambda x: x.name)))
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -22,10 +22,9 @@ class MetadataCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def metadata(cls, remote_path: str, password: str | None) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
remote = remote_path.split(":", 1)[0] if ":" in remote_path else ""
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_path, password)
|
||||
folder, laboratory = cls._resolve_folder(connection, remote_path, password)
|
||||
folder_api = FoldersApi(connection)
|
||||
metadata = folder_api.metadata(folder.id)
|
||||
print(json.dumps(metadata, ensure_ascii=False))
|
||||
|
||||
@@ -6,6 +6,7 @@ from unicodedata import normalize
|
||||
from mdrsclient.api import FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models.file import find_file
|
||||
|
||||
|
||||
class MkdirCommand(BaseCommand):
|
||||
@@ -22,14 +23,15 @@ class MkdirCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def mkdir(cls, remote_path: str) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
||||
if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None:
|
||||
files = cls._find_files(connection, parent_folder.id)
|
||||
if parent_folder.find_sub_folder(r_basename) is not None or find_file(files, r_basename) is not None:
|
||||
raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
|
||||
folder_api = FoldersApi(connection)
|
||||
folder_api.create(normalize("NFC", r_basename), parent_folder.id)
|
||||
|
||||
@@ -6,6 +6,7 @@ from unicodedata import normalize
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models.file import find_file
|
||||
|
||||
|
||||
class MvCommand(BaseCommand):
|
||||
@@ -24,8 +25,8 @@ class MvCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def mv(cls, src_path: str, dest_path: str) -> None:
|
||||
(s_remote, s_laboratory_name, s_path) = cls._parse_remote_host_with_path(src_path)
|
||||
(d_remote, d_laboratory_name, d_path) = cls._parse_remote_host_with_path(dest_path)
|
||||
s_remote, s_laboratory_name, s_path = cls._parse_remote_host_with_path(src_path)
|
||||
d_remote, d_laboratory_name, d_path = cls._parse_remote_host_with_path(dest_path)
|
||||
if s_remote != d_remote:
|
||||
raise IllegalArgumentException("Remote host mismatched.")
|
||||
if s_laboratory_name != d_laboratory_name:
|
||||
@@ -42,11 +43,13 @@ class MvCommand(BaseCommand):
|
||||
connection = cls._create_connection(s_remote)
|
||||
laboratory = cls._find_laboratory(connection, s_laboratory_name)
|
||||
s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
|
||||
s_parent_files = cls._find_files(connection, s_parent_folder.id)
|
||||
d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
|
||||
s_file = s_parent_folder.find_file(s_basename)
|
||||
d_parent_files = cls._find_files(connection, d_parent_folder.id)
|
||||
s_file = find_file(s_parent_files, s_basename)
|
||||
if s_file is not None:
|
||||
# source is file
|
||||
d_file = d_parent_folder.find_file(d_basename)
|
||||
d_file = find_file(d_parent_files, d_basename)
|
||||
if d_file is not None:
|
||||
raise IllegalArgumentException(f"File `{d_basename}` already exists.")
|
||||
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||
@@ -60,7 +63,7 @@ class MvCommand(BaseCommand):
|
||||
if s_folder is None:
|
||||
raise IllegalArgumentException(f"File or folder `{s_basename}` not found.")
|
||||
# source is folder
|
||||
if d_parent_folder.find_file(d_basename) is not None:
|
||||
if find_file(d_parent_files, d_basename) is not None:
|
||||
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
|
||||
d_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||
if d_folder is not None:
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Any
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models.file import find_file
|
||||
|
||||
|
||||
class RmCommand(BaseCommand):
|
||||
@@ -25,14 +26,15 @@ class RmCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def rm(cls, remote_path: str, is_recursive: bool) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
||||
file = parent_folder.find_file(r_basename)
|
||||
parent_files = cls._find_files(connection, parent_folder.id)
|
||||
file = find_file(parent_files, r_basename)
|
||||
if file is not None:
|
||||
file_api = FilesApi(connection)
|
||||
file_api.destroy(file)
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from argparse import Namespace
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
@@ -9,13 +10,15 @@ from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, MDRSException
|
||||
from mdrsclient.models import Folder
|
||||
from mdrsclient.models import File, Folder
|
||||
from mdrsclient.models.file import find_file
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UploadFileInfo:
|
||||
folder: Folder
|
||||
files: list[File]
|
||||
path: str
|
||||
|
||||
|
||||
@@ -46,13 +49,14 @@ class UploadCommand(BaseCommand):
|
||||
|
||||
@classmethod
|
||||
def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
remote, laboratory_name, r_path = cls._parse_remote_host_with_path(remote_path)
|
||||
l_path = os.path.abspath(local_path)
|
||||
if not os.path.exists(l_path):
|
||||
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
||||
connection = cls._create_connection(remote)
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
folder = cls._find_folder(connection, laboratory, r_path)
|
||||
files = cls._find_files(connection, folder.id)
|
||||
infos: list[UploadFileInfo] = []
|
||||
if os.path.isdir(l_path):
|
||||
if not is_recursive:
|
||||
@@ -60,6 +64,8 @@ class UploadCommand(BaseCommand):
|
||||
folder_api = FoldersApi(connection)
|
||||
folder_map: dict[str, Folder] = {}
|
||||
folder_map[r_path] = folder
|
||||
files_map: dict[str, list[File]] = {}
|
||||
files_map[r_path] = files
|
||||
l_basename = os.path.basename(l_path)
|
||||
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
||||
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
||||
@@ -68,23 +74,29 @@ class UploadCommand(BaseCommand):
|
||||
# prepare destination parent path
|
||||
d_parent_dirname = os.path.dirname(d_dirname)
|
||||
if folder_map.get(d_parent_dirname) is None:
|
||||
folder_map[d_parent_dirname] = cls._find_folder(connection, laboratory, d_parent_dirname)
|
||||
parent_folder = cls._find_folder(connection, laboratory, d_parent_dirname)
|
||||
folder_map[d_parent_dirname] = parent_folder
|
||||
parent_files = cls._find_files(connection, parent_folder.id)
|
||||
files_map[d_parent_dirname] = parent_files
|
||||
# prepare destination path
|
||||
if folder_map.get(d_dirname) is None:
|
||||
d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
|
||||
if d_folder is None:
|
||||
d_folder_id = folder_api.create(d_basename, folder_map[d_parent_dirname].id)
|
||||
d_folder_id = folder_api.create(normalize("NFC", d_basename), folder_map[d_parent_dirname].id)
|
||||
else:
|
||||
d_folder_id = d_folder.id
|
||||
print(d_dirname)
|
||||
folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
|
||||
files_map[d_dirname] = cls._find_files(connection, d_folder_id)
|
||||
if d_folder is None:
|
||||
folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
|
||||
# register upload file list
|
||||
for filename in filenames:
|
||||
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
||||
infos.append(
|
||||
UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
|
||||
)
|
||||
else:
|
||||
infos.append(UploadFileInfo(folder, l_path))
|
||||
infos.append(UploadFileInfo(folder, files, l_path))
|
||||
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
||||
|
||||
@classmethod
|
||||
@@ -98,7 +110,7 @@ class UploadCommand(BaseCommand):
|
||||
@classmethod
|
||||
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||
basename = os.path.basename(info.path)
|
||||
file = info.folder.find_file(basename)
|
||||
file = find_file(info.files, basename)
|
||||
try:
|
||||
if file is None:
|
||||
file_api.create(info.folder.id, info.path)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from mdrsclient.__version__ import __version__
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
|
||||
|
||||
class VersionCommand(BaseCommand):
|
||||
@classmethod
|
||||
def register(cls, parsers: Any) -> None:
|
||||
version_parser = parsers.add_parser("version", help="show the version of this tool")
|
||||
version_parser.set_defaults(func=cls.func)
|
||||
|
||||
@classmethod
|
||||
def func(cls, args: Namespace) -> None:
|
||||
cls.version()
|
||||
|
||||
@classmethod
|
||||
def version(cls) -> None:
|
||||
print(f"mdrs {__version__}")
|
||||
@@ -92,6 +92,8 @@ class MDRSConnection:
|
||||
self.__cache.laboratories = laboratories
|
||||
|
||||
def __build_url(self, path: str) -> str:
|
||||
if path.startswith("http://") or path.startswith("https://"):
|
||||
return path
|
||||
if self.url == "":
|
||||
raise MissingConfigurationException("remote host is not configured")
|
||||
return f"{self.url}/{path}"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import Any
|
||||
from unicodedata import normalize
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
@@ -25,3 +26,8 @@ class File:
|
||||
@property
|
||||
def updated_at_name(self) -> str:
|
||||
return iso8601_to_user_friendly(self.updated_at)
|
||||
|
||||
|
||||
def find_file(files: list[File], name: str) -> File | None:
|
||||
_name = normalize("NFC", name).lower()
|
||||
return next((x for x in files if x.name.lower() == _name), None)
|
||||
|
||||
@@ -3,7 +3,6 @@ from unicodedata import normalize
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
from mdrsclient.models.file import File
|
||||
from mdrsclient.models.utils import iso8601_to_user_friendly
|
||||
|
||||
|
||||
@@ -78,13 +77,8 @@ class FolderSimple:
|
||||
class Folder(FolderSimple):
|
||||
metadata: list[dict[str, Any]]
|
||||
sub_folders: list[FolderSimple]
|
||||
files: list[File]
|
||||
path: str
|
||||
|
||||
def find_sub_folder(self, name: str) -> FolderSimple | None:
|
||||
_name = normalize("NFC", name).lower()
|
||||
return next((x for x in self.sub_folders if x.name.lower() == _name), None)
|
||||
|
||||
def find_file(self, name: str) -> File | None:
|
||||
_name = normalize("NFC", name).lower()
|
||||
return next((x for x in self.files if x.name.lower() == _name), None)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
from typing import IO, Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
@@ -21,3 +22,10 @@ class FileLock:
|
||||
msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
elif os.name == "posix":
|
||||
fcntl.flock(file.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def page_num_from_url(url: str) -> int | None:
|
||||
parsed_url = urlparse(url)
|
||||
params = parse_qs(parsed_url.query)
|
||||
page = params.get("page", [None])[0]
|
||||
return int(page) if page is not None else None
|
||||
|
||||
+13
-12
@@ -1,12 +1,12 @@
|
||||
[tool.poetry]
|
||||
name = "mdrs-client-python"
|
||||
version = "1.3.10"
|
||||
version = "1.3.16"
|
||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
classifiers=[
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Development Status :: 4 - Beta",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Science/Research",
|
||||
@@ -14,6 +14,7 @@ classifiers=[
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"OSI Approved :: MIT License",
|
||||
"Topic :: Utilities",
|
||||
]
|
||||
@@ -23,20 +24,20 @@ packages = [
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.10"
|
||||
requests = "^2.32.3"
|
||||
requests = "^2.34.2"
|
||||
requests-toolbelt = "^1.0.0"
|
||||
python-dotenv = "^1.0.1"
|
||||
pydantic = "^2.10.4"
|
||||
pydantic-settings = "^2.7.0"
|
||||
PyJWT = "^2.10.1"
|
||||
validators = "^0.34.0"
|
||||
python-dotenv = "^1.1.0"
|
||||
pydantic = "^2.13.4"
|
||||
pydantic-settings = "^2.14.1"
|
||||
PyJWT = "^2.13.0"
|
||||
validators = "^0.35.0"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^24.10.0"
|
||||
flake8 = "^7.1.1"
|
||||
black = "^26.5.1"
|
||||
flake8 = "^7.2.0"
|
||||
Flake8-pyproject = "^1.2.3"
|
||||
isort = "^5.13.2"
|
||||
pyright = "^1.1.386"
|
||||
isort = "^8.0.1"
|
||||
pyright = "^1.1.401"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
mdrs = 'mdrsclient.__main__:main'
|
||||
|
||||
Reference in New Issue
Block a user