Compare commits
28 Commits
Author | SHA1 | Date | |
---|---|---|---|
7b3f1f2d09 | |||
8a5e1b68b7 | |||
dd00973bea | |||
0e5685d5ea | |||
6a2810f603 | |||
d5ac5cd427 | |||
ab7cd1b885 | |||
f2f898c263 | |||
24172dc65c | |||
f2c5a06cb4 | |||
49cb411af4 | |||
3e8ab8de3a | |||
d392379235 | |||
c8b16939d7 | |||
0d8deb02d7 | |||
c2a67aa861 | |||
4696b9799c | |||
020ef8835a | |||
55265e69a4 | |||
dbfc68c396 | |||
c3e2dfbd8e | |||
ce0a608db2 | |||
219858e0b6 | |||
a281a97b3e | |||
64f64b82dc | |||
9284346153 | |||
292ca1df27 | |||
f10b42a1f2 |
20
.cspell.json
20
.cspell.json
@ -1,8 +1,24 @@
|
|||||||
{
|
{
|
||||||
"version": "0.2",
|
"version": "0.2",
|
||||||
"language": "en,en-gb",
|
"language": "en,en-gb",
|
||||||
"ignoreWords": ["getframe", "pydantic", "UNLCK"],
|
"ignoreWords": [
|
||||||
"words": ["chacl", "mdrs", "mdrsclient", "neurodata", "Neuroinformatics", "RIKEN"],
|
"followlinks",
|
||||||
|
"getframe",
|
||||||
|
"pycache",
|
||||||
|
"pydantic",
|
||||||
|
"toolbelt",
|
||||||
|
"UNLCK"
|
||||||
|
],
|
||||||
|
"words": [
|
||||||
|
"chacl",
|
||||||
|
"kikan",
|
||||||
|
"mdrs",
|
||||||
|
"mdrsclient",
|
||||||
|
"neurodata",
|
||||||
|
"Neuroinformatics",
|
||||||
|
"orcid",
|
||||||
|
"RIKEN"
|
||||||
|
],
|
||||||
"ignorePaths": [
|
"ignorePaths": [
|
||||||
".env",
|
".env",
|
||||||
"__pycache__"
|
"__pycache__"
|
||||||
|
3
.gitignore
vendored
3
.gitignore
vendored
@ -160,4 +160,5 @@ cython_debug/
|
|||||||
.idea/
|
.idea/
|
||||||
|
|
||||||
# mdrs-cli
|
# mdrs-cli
|
||||||
.neurodatacli.config
|
.neurodatacli.config
|
||||||
|
poetry.toml
|
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"editor.formatOnSave": true,
|
"editor.formatOnSave": true,
|
||||||
"editor.codeActionsOnSave": {
|
"editor.codeActionsOnSave": {
|
||||||
"source.organizeImports": true
|
"source.organizeImports": "explicit"
|
||||||
},
|
},
|
||||||
"[python]": {
|
"[python]": {
|
||||||
"editor.defaultFormatter": "ms-python.black-formatter"
|
"editor.defaultFormatter": "ms-python.black-formatter"
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS
|
Copyright (c) 2023- Neuroinformatics Unit, RIKEN CBS
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
97
README.md
97
README.md
@ -4,7 +4,7 @@ The mdrs-client-python is python library and a command-line client for up- and d
|
|||||||
|
|
||||||
## Installing
|
## Installing
|
||||||
|
|
||||||
```
|
```shell
|
||||||
poetry install
|
poetry install
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -14,141 +14,146 @@ poetry install
|
|||||||
|
|
||||||
Create remote host configuration
|
Create remote host configuration
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs config create neurodata https://neurodata.riken.jp/api
|
mdrs config create neurodata https://neurodata.riken.jp/api
|
||||||
```
|
```
|
||||||
|
|
||||||
### login
|
### login
|
||||||
|
|
||||||
Login to remote host
|
Login to remote host
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs login neurodata:
|
mdrs login neurodata:
|
||||||
Username: (enter your login name)
|
Username: (enter your login name)
|
||||||
Password: (enter your password)
|
Password: (enter your password)
|
||||||
|
|
||||||
|
mdrs login -u USERNAME -p PASSWORD neurodata:
|
||||||
```
|
```
|
||||||
|
|
||||||
### logout
|
### logout
|
||||||
|
|
||||||
Logout from remote host
|
Logout from remote host
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs logout neurodata:
|
mdrs logout neurodata:
|
||||||
```
|
```
|
||||||
|
|
||||||
### whoami
|
### whoami
|
||||||
|
|
||||||
Print current user name
|
Print current user name
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs whoami neurodata:
|
mdrs whoami neurodata:
|
||||||
```
|
```
|
||||||
|
|
||||||
### labs
|
### labs
|
||||||
|
|
||||||
List all laboratories
|
List all laboratories
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs labs neurodata:
|
mdrs labs neurodata:
|
||||||
```
|
```
|
||||||
|
|
||||||
### ls
|
### ls
|
||||||
|
|
||||||
List the folder contents
|
List the folder contents
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs ls neurodata:/NIU/Repository/
|
mdrs ls neurodata:/NIU/Repository/
|
||||||
$ mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||||
$ mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
||||||
$ mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
||||||
```
|
```
|
||||||
|
|
||||||
### mkdir
|
### mkdir
|
||||||
|
|
||||||
Create a new folder
|
Create a new folder
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs mkdir neurodata:/NIU/Repository/TEST
|
mdrs mkdir neurodata:/NIU/Repository/TEST
|
||||||
```
|
```
|
||||||
|
|
||||||
### upload
|
### upload
|
||||||
|
|
||||||
Upload the file or directory
|
Upload the file or directory
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
|
mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
|
||||||
$ mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
|
mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
|
||||||
|
mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
|
||||||
```
|
```
|
||||||
|
|
||||||
### download
|
### download
|
||||||
|
|
||||||
Download the file or folder
|
Download the file or folder
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
||||||
$ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||||
$ mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||||
|
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||||
|
mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||||
```
|
```
|
||||||
|
|
||||||
### mv
|
### mv
|
||||||
|
|
||||||
Move or rename the file or folder
|
Move or rename the file or folder
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||||
$ mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||||
```
|
```
|
||||||
|
|
||||||
### cp
|
### cp
|
||||||
|
|
||||||
Copy the file and folder
|
Copy the file and folder
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||||
$ mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||||
```
|
```
|
||||||
|
|
||||||
### rm
|
### rm
|
||||||
|
|
||||||
Remove the file or folder
|
Remove the file or folder
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat
|
mdrs rm neurodata:/NIU/Repository/TEST/sample.dat
|
||||||
$ mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset
|
mdrs rm -r neurodata:/NIU/Repository/TEST/dataset
|
||||||
```
|
```
|
||||||
|
|
||||||
### chacl
|
### chacl
|
||||||
|
|
||||||
Change the folder access level
|
Change the folder access level
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs chacl private neurodata:/NIU/Repository/Private
|
mdrs chacl private neurodata:/NIU/Repository/Private
|
||||||
$ mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
|
mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
|
||||||
$ mdrs chacl pw_open -r -p FOLDER_PASSWORD neurodata:/NIU/Repository/PW_Open
|
mdrs chacl pw_open -r -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||||
```
|
```
|
||||||
|
|
||||||
### metadata
|
### metadata
|
||||||
|
|
||||||
Get a folder metadata
|
Get a folder metadata
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs metadata neurodata:/NIU/Repository/TEST/
|
mdrs metadata neurodata:/NIU/Repository/TEST/
|
||||||
$ mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||||
```
|
```
|
||||||
|
|
||||||
### file-metadata
|
### file-metadata
|
||||||
|
|
||||||
Get the file metadata
|
Get the file metadata
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
||||||
$ mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
### help
|
### help
|
||||||
|
|
||||||
Show the help message and exit
|
Show the help message and exit
|
||||||
|
|
||||||
```
|
```shell
|
||||||
$ mdrs -h
|
mdrs -h
|
||||||
```
|
```
|
||||||
|
@ -1 +1 @@
|
|||||||
1.2.0
|
1.3.11
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import sys
|
import sys
|
||||||
|
from json import JSONDecodeError
|
||||||
|
|
||||||
from mdrsclient.commands import (
|
from mdrsclient.commands import (
|
||||||
ChaclCommand,
|
ChaclCommand,
|
||||||
@ -52,6 +53,9 @@ def main() -> None:
|
|||||||
except MDRSException as e:
|
except MDRSException as e:
|
||||||
print(f"Error: {e}")
|
print(f"Error: {e}")
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
except JSONDecodeError:
|
||||||
|
print("Unexpected response returned. Please check the configuration or the server's operational status.")
|
||||||
|
sys.exit(2)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
sys.exit(130)
|
sys.exit(130)
|
||||||
|
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
from mdrsclient.api.file import FileApi
|
from mdrsclient.api.files import FilesApi
|
||||||
from mdrsclient.api.folder import FolderApi
|
from mdrsclient.api.folders import FoldersApi
|
||||||
from mdrsclient.api.laboratory import LaboratoryApi
|
from mdrsclient.api.laboratories import LaboratoriesApi
|
||||||
from mdrsclient.api.user import UserApi
|
from mdrsclient.api.users import UsersApi
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"FileApi",
|
"FilesApi",
|
||||||
"FolderApi",
|
"FoldersApi",
|
||||||
"LaboratoryApi",
|
"LaboratoriesApi",
|
||||||
"UserApi",
|
"UsersApi",
|
||||||
]
|
]
|
||||||
|
@ -1,7 +1,10 @@
|
|||||||
|
import mimetypes
|
||||||
|
import os
|
||||||
from typing import Any, Final
|
from typing import Any, Final
|
||||||
|
|
||||||
from pydantic import TypeAdapter
|
from pydantic import TypeAdapter
|
||||||
from pydantic.dataclasses import dataclass
|
from pydantic.dataclasses import dataclass
|
||||||
|
from requests_toolbelt.multipart.encoder import MultipartEncoder
|
||||||
|
|
||||||
from mdrsclient.api.base import BaseApi
|
from mdrsclient.api.base import BaseApi
|
||||||
from mdrsclient.api.utils import token_check
|
from mdrsclient.api.utils import token_check
|
||||||
@ -10,12 +13,13 @@ from mdrsclient.models import File
|
|||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class FileCreateResponse:
|
class FilesApiCreateResponse:
|
||||||
id: str
|
id: str
|
||||||
|
|
||||||
|
|
||||||
class FileApi(BaseApi):
|
class FilesApi(BaseApi):
|
||||||
ENTRYPOINT: Final[str] = "v2/file/"
|
ENTRYPOINT: Final[str] = "v3/files/"
|
||||||
|
FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
|
||||||
|
|
||||||
def retrieve(self, id: str) -> File:
|
def retrieve(self, id: str) -> File:
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
@ -29,30 +33,43 @@ class FileApi(BaseApi):
|
|||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
url = self.ENTRYPOINT
|
url = self.ENTRYPOINT
|
||||||
token_check(self.connection)
|
token_check(self.connection)
|
||||||
data: dict[str, str | int] = {"folder_id": folder_id}
|
data: dict[str, str | int] | MultipartEncoder = {}
|
||||||
try:
|
try:
|
||||||
with open(path, mode="rb") as fp:
|
with open(os.path.realpath(path), mode="rb") as fp:
|
||||||
response = self.connection.post(url, data=data, files={"file": fp})
|
data = MultipartEncoder(
|
||||||
|
fields={"folder_id": folder_id, "file": (os.path.basename(path), fp, self._get_mime_type(path))}
|
||||||
|
)
|
||||||
|
response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
|
||||||
self._raise_response_error(response)
|
self._raise_response_error(response)
|
||||||
ret = TypeAdapter(FileCreateResponse).validate_python(response.json())
|
ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
|
||||||
except OSError:
|
except OSError:
|
||||||
raise UnexpectedException(f"Could not open `{path}` file.")
|
raise UnexpectedException(f"Could not open `{path}` file.")
|
||||||
|
except MemoryError:
|
||||||
|
raise UnexpectedException("Out of memory.")
|
||||||
|
except Exception as e:
|
||||||
|
raise UnexpectedException("Unspecified error.") from e
|
||||||
return ret.id
|
return ret.id
|
||||||
|
|
||||||
def update(self, file: File, path: str | None) -> bool:
|
def update(self, file: File, path: str | None) -> bool:
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
url = self.ENTRYPOINT + file.id + "/"
|
url = self.ENTRYPOINT + file.id + "/"
|
||||||
token_check(self.connection)
|
token_check(self.connection)
|
||||||
|
data: dict[str, str | int] | MultipartEncoder = {}
|
||||||
if path is not None:
|
if path is not None:
|
||||||
# update file body
|
# update file body
|
||||||
try:
|
try:
|
||||||
with open(path, mode="rb") as fp:
|
with open(os.path.realpath(path), mode="rb") as fp:
|
||||||
response = self.connection.put(url, files={"file": fp})
|
data = MultipartEncoder(fields={"file": (os.path.basename(path), fp, self._get_mime_type(path))})
|
||||||
|
response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
|
||||||
except OSError:
|
except OSError:
|
||||||
raise UnexpectedException(f"Could not open `{path}` file.")
|
raise UnexpectedException(f"Could not open `{path}` file.")
|
||||||
|
except MemoryError:
|
||||||
|
raise UnexpectedException("Out of memory.")
|
||||||
|
except Exception as e:
|
||||||
|
raise UnexpectedException("Unspecified error.") from e
|
||||||
else:
|
else:
|
||||||
# update metadata
|
# update metadata
|
||||||
data: dict[str, str | int] = {"name": file.name, "description": file.description}
|
data = {"name": file.name, "description": file.description}
|
||||||
response = self.connection.put(url, data=data)
|
response = self.connection.put(url, data=data)
|
||||||
self._raise_response_error(response)
|
self._raise_response_error(response)
|
||||||
return True
|
return True
|
||||||
@ -93,7 +110,8 @@ class FileApi(BaseApi):
|
|||||||
|
|
||||||
def download(self, file: File, path: str) -> bool:
|
def download(self, file: File, path: str) -> bool:
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
url = "v2/" + file.download_url
|
url = file.download_url
|
||||||
|
token_check(self.connection)
|
||||||
response = self.connection.get(url, stream=True)
|
response = self.connection.get(url, stream=True)
|
||||||
self._raise_response_error(response)
|
self._raise_response_error(response)
|
||||||
try:
|
try:
|
||||||
@ -105,3 +123,9 @@ class FileApi(BaseApi):
|
|||||||
except PermissionError:
|
except PermissionError:
|
||||||
print(f"Cannot create file `{path}`: Permission denied.")
|
print(f"Cannot create file `{path}`: Permission denied.")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _get_mime_type(self, path: str) -> str:
|
||||||
|
mt = mimetypes.guess_type(path)
|
||||||
|
if mt:
|
||||||
|
return mt[0] or self.FALLBACK_MIMETYPE
|
||||||
|
return self.FALLBACK_MIMETYPE
|
@ -11,12 +11,12 @@ from mdrsclient.models import Folder, FolderSimple
|
|||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class FolderCreateResponse:
|
class FoldersApiCreateResponse:
|
||||||
id: str
|
id: str
|
||||||
|
|
||||||
|
|
||||||
class FolderApi(BaseApi):
|
class FoldersApi(BaseApi):
|
||||||
ENTRYPOINT: Final[str] = "v2/folder/"
|
ENTRYPOINT: Final[str] = "v3/folders/"
|
||||||
|
|
||||||
def list(self, laboratory_id: int, path: str) -> list[FolderSimple]:
|
def list(self, laboratory_id: int, path: str) -> list[FolderSimple]:
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
@ -46,7 +46,7 @@ class FolderApi(BaseApi):
|
|||||||
token_check(self.connection)
|
token_check(self.connection)
|
||||||
response = self.connection.post(url, data=data)
|
response = self.connection.post(url, data=data)
|
||||||
self._raise_response_error(response)
|
self._raise_response_error(response)
|
||||||
ret = TypeAdapter(FolderCreateResponse).validate_python(response.json())
|
ret = TypeAdapter(FoldersApiCreateResponse).validate_python(response.json())
|
||||||
return ret.id
|
return ret.id
|
||||||
|
|
||||||
def update(self, folder: FolderSimple) -> bool:
|
def update(self, folder: FolderSimple) -> bool:
|
@ -7,8 +7,8 @@ from mdrsclient.api.utils import token_check
|
|||||||
from mdrsclient.models import Laboratories, Laboratory
|
from mdrsclient.models import Laboratories, Laboratory
|
||||||
|
|
||||||
|
|
||||||
class LaboratoryApi(BaseApi):
|
class LaboratoriesApi(BaseApi):
|
||||||
ENTRYPOINT: Final[str] = "v2/laboratory/"
|
ENTRYPOINT: Final[str] = "v3/laboratories/"
|
||||||
|
|
||||||
def list(self) -> Laboratories:
|
def list(self) -> Laboratories:
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
@ -1,46 +0,0 @@
|
|||||||
from dataclasses import field
|
|
||||||
from typing import Final
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from pydantic import TypeAdapter
|
|
||||||
from pydantic.dataclasses import dataclass
|
|
||||||
|
|
||||||
from mdrsclient.api.base import BaseApi
|
|
||||||
from mdrsclient.exceptions import UnauthorizedException
|
|
||||||
from mdrsclient.models import Laboratory, Token, User
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class UserAuthResponse(Token):
|
|
||||||
is_reviewer: bool | None = None
|
|
||||||
laboratories: list[Laboratory] = field(default_factory=list)
|
|
||||||
|
|
||||||
|
|
||||||
class UserApi(BaseApi):
|
|
||||||
ENTRYPOINT: Final[str] = "v2/"
|
|
||||||
|
|
||||||
def auth(self, username: str, password: str) -> tuple[User, Token]:
|
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
||||||
url = self.ENTRYPOINT + "auth/"
|
|
||||||
data: dict[str, str | int] = {"username": username, "password": password}
|
|
||||||
response = self.connection.post(url, data=data)
|
|
||||||
if response.status_code == requests.codes.unauthorized:
|
|
||||||
raise UnauthorizedException("Invalid username or password.")
|
|
||||||
self._raise_response_error(response)
|
|
||||||
obj = TypeAdapter(UserAuthResponse).validate_python(response.json())
|
|
||||||
token = Token(access=obj.access, refresh=obj.refresh)
|
|
||||||
laboratory_ids = list(map(lambda x: x.id, obj.laboratories))
|
|
||||||
is_reviewer = obj.is_reviewer if obj.is_reviewer is not None else False
|
|
||||||
user = User(id=token.user_id, username=username, laboratory_ids=laboratory_ids, is_reviewer=is_reviewer)
|
|
||||||
return (user, token)
|
|
||||||
|
|
||||||
def refresh(self, token: Token) -> Token:
|
|
||||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
||||||
url = self.ENTRYPOINT + "refresh/"
|
|
||||||
data: dict[str, str | int] = {"refresh": token.refresh}
|
|
||||||
response = self.connection.post(url, data=data)
|
|
||||||
if response.status_code == requests.codes.unauthorized:
|
|
||||||
raise UnauthorizedException("Token is invalid or expired.")
|
|
||||||
self._raise_response_error(response)
|
|
||||||
token = TypeAdapter(Token).validate_python(response.json())
|
|
||||||
return token
|
|
69
mdrsclient/api/users.py
Normal file
69
mdrsclient/api/users.py
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
from typing import Final
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from pydantic import TypeAdapter
|
||||||
|
from pydantic.dataclasses import dataclass
|
||||||
|
|
||||||
|
from mdrsclient.api.base import BaseApi
|
||||||
|
from mdrsclient.exceptions import UnauthorizedException
|
||||||
|
from mdrsclient.models import Token, User
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UsersCurrentResponseLaboratory:
|
||||||
|
id: int
|
||||||
|
name: str
|
||||||
|
role: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UsersApiCurrentResponse:
|
||||||
|
id: int
|
||||||
|
username: str
|
||||||
|
first_name: str
|
||||||
|
last_name: str
|
||||||
|
email: str
|
||||||
|
orcid_id: str
|
||||||
|
laboratories: list[UsersCurrentResponseLaboratory]
|
||||||
|
is_staff: bool
|
||||||
|
is_active: bool
|
||||||
|
is_superuser: bool
|
||||||
|
is_reviewer: bool
|
||||||
|
last_login: str # ISO8601
|
||||||
|
date_joined: str # ISO8601
|
||||||
|
|
||||||
|
|
||||||
|
class UsersApi(BaseApi):
|
||||||
|
ENTRYPOINT: Final[str] = "v3/users/"
|
||||||
|
|
||||||
|
def current(self) -> User:
|
||||||
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
|
url = self.ENTRYPOINT + "current/"
|
||||||
|
response = self.connection.get(url)
|
||||||
|
self._raise_response_error(response)
|
||||||
|
obj = TypeAdapter(UsersApiCurrentResponse).validate_python(response.json())
|
||||||
|
laboratory_ids = list(map(lambda x: x.id, obj.laboratories))
|
||||||
|
user = User(id=obj.id, username=obj.username, laboratory_ids=laboratory_ids, is_reviewer=obj.is_reviewer)
|
||||||
|
return user
|
||||||
|
|
||||||
|
def token(self, username: str, password: str) -> Token:
|
||||||
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
|
url = self.ENTRYPOINT + "token/"
|
||||||
|
data: dict[str, str | int] = {"username": username, "password": password}
|
||||||
|
response = self.connection.post(url, data=data)
|
||||||
|
if response.status_code == requests.codes.unauthorized:
|
||||||
|
raise UnauthorizedException("Invalid username or password.")
|
||||||
|
self._raise_response_error(response)
|
||||||
|
token = TypeAdapter(Token).validate_python(response.json())
|
||||||
|
return token
|
||||||
|
|
||||||
|
def tokenRefresh(self, token: Token) -> Token:
|
||||||
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||||
|
url = self.ENTRYPOINT + "token/refresh/"
|
||||||
|
data: dict[str, str | int] = {"refresh": token.refresh}
|
||||||
|
response = self.connection.post(url, data=data)
|
||||||
|
if response.status_code == requests.codes.unauthorized:
|
||||||
|
raise UnauthorizedException("Token is invalid or expired.")
|
||||||
|
self._raise_response_error(response)
|
||||||
|
token = TypeAdapter(Token).validate_python(response.json())
|
||||||
|
return token
|
@ -1,4 +1,4 @@
|
|||||||
from mdrsclient.api.user import UserApi
|
from mdrsclient.api.users import UsersApi
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
from mdrsclient.exceptions import UnauthorizedException
|
from mdrsclient.exceptions import UnauthorizedException
|
||||||
|
|
||||||
@ -8,9 +8,9 @@ def token_check(connection: MDRSConnection) -> None:
|
|||||||
connection.lock.acquire()
|
connection.lock.acquire()
|
||||||
if connection.token is not None:
|
if connection.token is not None:
|
||||||
if connection.token.is_refresh_required:
|
if connection.token.is_refresh_required:
|
||||||
user_api = UserApi(connection)
|
user_api = UsersApi(connection)
|
||||||
try:
|
try:
|
||||||
connection.token = user_api.refresh(connection.token)
|
connection.token = user_api.tokenRefresh(connection.token)
|
||||||
except UnauthorizedException:
|
except UnauthorizedException:
|
||||||
connection.logout()
|
connection.logout()
|
||||||
elif connection.token.is_expired:
|
elif connection.token.is_expired:
|
||||||
|
@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from unicodedata import normalize
|
from unicodedata import normalize
|
||||||
|
|
||||||
from mdrsclient.api import FolderApi, LaboratoryApi
|
from mdrsclient.api import FoldersApi, LaboratoriesApi
|
||||||
from mdrsclient.config import ConfigFile
|
from mdrsclient.config import ConfigFile
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
from mdrsclient.exceptions import (
|
from mdrsclient.exceptions import (
|
||||||
@ -31,7 +31,7 @@ class BaseCommand(ABC):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def _find_laboratory(cls, connection: MDRSConnection, name: str) -> Laboratory:
|
def _find_laboratory(cls, connection: MDRSConnection, name: str) -> Laboratory:
|
||||||
if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired:
|
if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired:
|
||||||
laboratory_api = LaboratoryApi(connection)
|
laboratory_api = LaboratoriesApi(connection)
|
||||||
connection.laboratories = laboratory_api.list()
|
connection.laboratories = laboratory_api.list()
|
||||||
laboratory = connection.laboratories.find_by_name(name)
|
laboratory = connection.laboratories.find_by_name(name)
|
||||||
if laboratory is None:
|
if laboratory is None:
|
||||||
@ -42,7 +42,7 @@ class BaseCommand(ABC):
|
|||||||
def _find_folder(
|
def _find_folder(
|
||||||
cls, connection: MDRSConnection, laboratory: Laboratory, path: str, password: str | None = None
|
cls, connection: MDRSConnection, laboratory: Laboratory, path: str, password: str | None = None
|
||||||
) -> Folder:
|
) -> Folder:
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
folders = folder_api.list(laboratory.id, normalize("NFC", path))
|
folders = folder_api.list(laboratory.id, normalize("NFC", path))
|
||||||
if len(folders) != 1:
|
if len(folders) != 1:
|
||||||
raise UnexpectedException(f"Folder `{path}` not found.")
|
raise UnexpectedException(f"Folder `{path}` not found.")
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import FolderApi
|
from mdrsclient.api import FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
from mdrsclient.models import FolderAccessLevel
|
from mdrsclient.models import FolderAccessLevel
|
||||||
@ -36,5 +36,5 @@ class ChaclCommand(BaseCommand):
|
|||||||
connection = cls._create_connection(remote)
|
connection = cls._create_connection(remote)
|
||||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||||
folder = cls._find_folder(connection, laboratory, r_path)
|
folder = cls._find_folder(connection, laboratory, r_path)
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
folder_api.acl(folder.id, access_level, is_recursive, password)
|
folder_api.acl(folder.id, access_level, is_recursive, password)
|
||||||
|
@ -25,11 +25,11 @@ class ConfigCommand(BaseCommand):
|
|||||||
update_parser.add_argument("url", help="API entrypoint url of remote host")
|
update_parser.add_argument("url", help="API entrypoint url of remote host")
|
||||||
update_parser.set_defaults(func=cls.func_update)
|
update_parser.set_defaults(func=cls.func_update)
|
||||||
# config list
|
# config list
|
||||||
list_parser = config_parsers.add_parser("list", help="list all the remote hosts")
|
list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
|
||||||
list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
|
list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
|
||||||
list_parser.set_defaults(func=cls.func_list)
|
list_parser.set_defaults(func=cls.func_list)
|
||||||
# config delete
|
# config delete
|
||||||
delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host")
|
delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host", aliases=["remove"])
|
||||||
delete_parser.add_argument("remote", help="label of remote host")
|
delete_parser.add_argument("remote", help="label of remote host")
|
||||||
delete_parser.set_defaults(func=cls.func_delete)
|
delete_parser.set_defaults(func=cls.func_delete)
|
||||||
|
|
||||||
|
@ -3,7 +3,7 @@ from argparse import Namespace
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from unicodedata import normalize
|
from unicodedata import normalize
|
||||||
|
|
||||||
from mdrsclient.api import FileApi, FolderApi
|
from mdrsclient.api import FilesApi, FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
|
|
||||||
@ -56,7 +56,7 @@ class CpCommand(BaseCommand):
|
|||||||
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||||
if d_sub_folder is not None:
|
if d_sub_folder is not None:
|
||||||
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
|
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
|
||||||
file_api = FileApi(connection)
|
file_api = FilesApi(connection)
|
||||||
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
||||||
file_api.copy(s_file, d_parent_folder.id, normalize("NFC", d_basename))
|
file_api.copy(s_file, d_parent_folder.id, normalize("NFC", d_basename))
|
||||||
else:
|
else:
|
||||||
@ -73,6 +73,6 @@ class CpCommand(BaseCommand):
|
|||||||
if d_folder.id == s_folder.id:
|
if d_folder.id == s_folder.id:
|
||||||
raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
|
raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
|
||||||
raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
|
raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
if s_parent_folder.id != d_parent_folder.id or s_basename != d_basename:
|
if s_parent_folder.id != d_parent_folder.id or s_basename != d_basename:
|
||||||
folder_api.copy(s_folder, d_parent_folder.id, normalize("NFC", d_basename))
|
folder_api.copy(s_folder, d_parent_folder.id, normalize("NFC", d_basename))
|
||||||
|
@ -5,11 +5,11 @@ from typing import Any
|
|||||||
|
|
||||||
from pydantic.dataclasses import dataclass
|
from pydantic.dataclasses import dataclass
|
||||||
|
|
||||||
from mdrsclient.api import FileApi, FolderApi
|
from mdrsclient.api import FilesApi, FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
|
||||||
from mdrsclient.models import File
|
from mdrsclient.models import File, Folder, Laboratory
|
||||||
from mdrsclient.settings import CONCURRENT
|
from mdrsclient.settings import CONCURRENT
|
||||||
|
|
||||||
|
|
||||||
@ -19,6 +19,13 @@ class DownloadFileInfo:
|
|||||||
path: str
|
path: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DownloadContext:
|
||||||
|
hasError: bool
|
||||||
|
isSkipIfExists: bool
|
||||||
|
files: list[DownloadFileInfo]
|
||||||
|
|
||||||
|
|
||||||
class DownloadCommand(BaseCommand):
|
class DownloadCommand(BaseCommand):
|
||||||
@classmethod
|
@classmethod
|
||||||
def register(cls, parsers: Any) -> None:
|
def register(cls, parsers: Any) -> None:
|
||||||
@ -26,6 +33,15 @@ class DownloadCommand(BaseCommand):
|
|||||||
download_parser.add_argument(
|
download_parser.add_argument(
|
||||||
"-r", "--recursive", help="download folders and their contents recursive", action="store_true"
|
"-r", "--recursive", help="download folders and their contents recursive", action="store_true"
|
||||||
)
|
)
|
||||||
|
download_parser.add_argument(
|
||||||
|
"-s",
|
||||||
|
"--skip-if-exists",
|
||||||
|
help="skip the download if file is already downloaded and file size is the same",
|
||||||
|
action="store_true",
|
||||||
|
)
|
||||||
|
download_parser.add_argument(
|
||||||
|
"-e", "--exclude", help="exclude to download path matched file or folders", action="append"
|
||||||
|
)
|
||||||
download_parser.add_argument("-p", "--password", help="password to use when open locked folder")
|
download_parser.add_argument("-p", "--password", help="password to use when open locked folder")
|
||||||
download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)")
|
download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)")
|
||||||
download_parser.add_argument("local_path", help="local folder path (/foo/bar/)")
|
download_parser.add_argument("local_path", help="local folder path (/foo/bar/)")
|
||||||
@ -36,11 +52,21 @@ class DownloadCommand(BaseCommand):
|
|||||||
remote_path = str(args.remote_path)
|
remote_path = str(args.remote_path)
|
||||||
local_path = str(args.local_path)
|
local_path = str(args.local_path)
|
||||||
is_recursive = bool(args.recursive)
|
is_recursive = bool(args.recursive)
|
||||||
|
is_skip_if_exists = bool(args.skip_if_exists)
|
||||||
password = str(args.password) if args.password else None
|
password = str(args.password) if args.password else None
|
||||||
cls.download(remote_path, local_path, is_recursive, password)
|
excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
|
||||||
|
cls.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def download(cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None) -> None:
|
def download(
|
||||||
|
cls,
|
||||||
|
remote_path: str,
|
||||||
|
local_path: str,
|
||||||
|
is_recursive: bool,
|
||||||
|
is_skip_if_exists: bool,
|
||||||
|
password: str | None,
|
||||||
|
excludes: list[str],
|
||||||
|
) -> None:
|
||||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||||
r_path = r_path.rstrip("/")
|
r_path = r_path.rstrip("/")
|
||||||
r_dirname = os.path.dirname(r_path)
|
r_dirname = os.path.dirname(r_path)
|
||||||
@ -52,42 +78,81 @@ class DownloadCommand(BaseCommand):
|
|||||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||||
r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||||
file = r_parent_folder.find_file(r_basename)
|
file = r_parent_folder.find_file(r_basename)
|
||||||
download_files: list[DownloadFileInfo] = []
|
|
||||||
if file is not None:
|
if file is not None:
|
||||||
|
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||||
|
return
|
||||||
|
context = DownloadContext(False, is_skip_if_exists, [])
|
||||||
l_path = os.path.join(l_dirname, r_basename)
|
l_path = os.path.join(l_dirname, r_basename)
|
||||||
download_files.append(DownloadFileInfo(file, l_path))
|
context.files.append(DownloadFileInfo(file, l_path))
|
||||||
|
cls.__multiple_download(connection, context)
|
||||||
else:
|
else:
|
||||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||||
if folder is None:
|
if folder is None:
|
||||||
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
||||||
if not is_recursive:
|
if not is_recursive:
|
||||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
cls.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname)
|
cls.__multiple_download_pickup_recursive_files(
|
||||||
cls.__multiple_download(connection, download_files)
|
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_download_pickup_recursive_files(
|
def __multiple_download_pickup_recursive_files(
|
||||||
cls, folder_api: FolderApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str
|
cls,
|
||||||
|
connection: MDRSConnection,
|
||||||
|
folder_api: FoldersApi,
|
||||||
|
laboratory: Laboratory,
|
||||||
|
folder_id: str,
|
||||||
|
basedir: str,
|
||||||
|
excludes: list[str],
|
||||||
|
is_skip_if_exists: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
context = DownloadContext(False, is_skip_if_exists, [])
|
||||||
folder = folder_api.retrieve(folder_id)
|
folder = folder_api.retrieve(folder_id)
|
||||||
dirname = os.path.join(basedir, folder.name)
|
dirname = os.path.join(basedir, folder.name)
|
||||||
|
if cls.__check_excludes(excludes, laboratory, folder, None):
|
||||||
|
return
|
||||||
if not os.path.exists(dirname):
|
if not os.path.exists(dirname):
|
||||||
os.makedirs(dirname)
|
os.makedirs(dirname)
|
||||||
print(dirname)
|
print(dirname)
|
||||||
for file in folder.files:
|
for file in folder.files:
|
||||||
|
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||||
|
continue
|
||||||
path = os.path.join(dirname, file.name)
|
path = os.path.join(dirname, file.name)
|
||||||
infolist.append(DownloadFileInfo(file, path))
|
context.files.append(DownloadFileInfo(file, path))
|
||||||
|
cls.__multiple_download(connection, context)
|
||||||
|
if context.hasError:
|
||||||
|
raise UnexpectedException("Some files failed to download.")
|
||||||
for sub_folder in folder.sub_folders:
|
for sub_folder in folder.sub_folders:
|
||||||
cls.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname)
|
cls.__multiple_download_pickup_recursive_files(
|
||||||
|
connection, folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_download(cls, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None:
|
def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
|
||||||
file_api = FileApi(connection)
|
file_api = FilesApi(connection)
|
||||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||||
pool.map(lambda x: cls.__multiple_download_worker(file_api, x), infolist)
|
results = pool.map(
|
||||||
|
lambda x: cls.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
|
||||||
|
)
|
||||||
|
hasError = next(filter(lambda x: x is False, results), None)
|
||||||
|
if hasError is not None:
|
||||||
|
context.hasError = True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_download_worker(cls, file_api: FileApi, info: DownloadFileInfo) -> None:
|
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||||
file_api.download(info.file, info.path)
|
if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
|
||||||
|
try:
|
||||||
|
file_api.download(info.file, info.path)
|
||||||
|
except Exception:
|
||||||
|
print(f"Failed: ${info.path}")
|
||||||
|
if os.path.isfile(info.path):
|
||||||
|
os.remove(info.path)
|
||||||
|
return False
|
||||||
print(info.path)
|
print(info.path)
|
||||||
|
return True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
|
||||||
|
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
|
||||||
|
return path in excludes
|
||||||
|
@ -3,7 +3,7 @@ import os
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import FileApi
|
from mdrsclient.api import FilesApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
|
|
||||||
@ -34,6 +34,6 @@ class FileMetadataCommand(BaseCommand):
|
|||||||
file = folder.find_file(r_basename)
|
file = folder.find_file(r_basename)
|
||||||
if file is None:
|
if file is None:
|
||||||
raise IllegalArgumentException(f"File `{r_basename}` not found.")
|
raise IllegalArgumentException(f"File `{r_basename}` not found.")
|
||||||
file_api = FileApi(connection)
|
file_api = FilesApi(connection)
|
||||||
metadata = file_api.metadata(file)
|
metadata = file_api.metadata(file)
|
||||||
print(json.dumps(metadata, ensure_ascii=False))
|
print(json.dumps(metadata, ensure_ascii=False))
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import LaboratoryApi
|
from mdrsclient.api import LaboratoriesApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
|
|
||||||
|
|
||||||
@ -21,7 +21,7 @@ class LabsCommand(BaseCommand):
|
|||||||
def labs(cls, remote: str) -> None:
|
def labs(cls, remote: str) -> None:
|
||||||
remote = cls._parse_remote_host(remote)
|
remote = cls._parse_remote_host(remote)
|
||||||
connection = cls._create_connection(remote)
|
connection = cls._create_connection(remote)
|
||||||
laboratory_api = LaboratoryApi(connection)
|
laboratory_api = LaboratoriesApi(connection)
|
||||||
laboratories = laboratory_api.list()
|
laboratories = laboratory_api.list()
|
||||||
connection.laboratories = laboratories
|
connection.laboratories = laboratories
|
||||||
label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
|
label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
|
||||||
|
@ -2,7 +2,7 @@ import getpass
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import UserApi
|
from mdrsclient.api import UsersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.config import ConfigFile
|
from mdrsclient.config import ConfigFile
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
@ -13,14 +13,16 @@ class LoginCommand(BaseCommand):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def register(cls, parsers: Any) -> None:
|
def register(cls, parsers: Any) -> None:
|
||||||
login_parser = parsers.add_parser("login", help="login to remote host")
|
login_parser = parsers.add_parser("login", help="login to remote host")
|
||||||
|
login_parser.add_argument("-u", "--username", help="login username")
|
||||||
|
login_parser.add_argument("-p", "--password", help="login password")
|
||||||
login_parser.add_argument("remote", help="label of remote host")
|
login_parser.add_argument("remote", help="label of remote host")
|
||||||
login_parser.set_defaults(func=cls.func)
|
login_parser.set_defaults(func=cls.func)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def func(cls, args: Namespace) -> None:
|
def func(cls, args: Namespace) -> None:
|
||||||
remote = str(args.remote)
|
remote = str(args.remote)
|
||||||
username = input("Username: ").strip()
|
username = str(args.username) if args.password else input("Username: ").strip()
|
||||||
password = getpass.getpass("Password: ").strip()
|
password = str(args.password) if args.password else getpass.getpass("Password: ").strip()
|
||||||
cls.login(remote, username, password)
|
cls.login(remote, username, password)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -30,8 +32,9 @@ class LoginCommand(BaseCommand):
|
|||||||
if config.url is None:
|
if config.url is None:
|
||||||
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
|
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
|
||||||
connection = MDRSConnection(config.remote, config.url)
|
connection = MDRSConnection(config.remote, config.url)
|
||||||
user_api = UserApi(connection)
|
user_api = UsersApi(connection)
|
||||||
(user, token) = user_api.auth(username, password)
|
token = user_api.token(username, password)
|
||||||
print("Login Successful")
|
|
||||||
connection.user = user
|
|
||||||
connection.token = token
|
connection.token = token
|
||||||
|
user = user_api.current()
|
||||||
|
connection.user = user
|
||||||
|
print("Login Successful")
|
||||||
|
@ -4,7 +4,7 @@ from typing import Any
|
|||||||
|
|
||||||
from pydantic.dataclasses import dataclass
|
from pydantic.dataclasses import dataclass
|
||||||
|
|
||||||
from mdrsclient.api import FolderApi
|
from mdrsclient.api import FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
from mdrsclient.exceptions import UnauthorizedException
|
from mdrsclient.exceptions import UnauthorizedException
|
||||||
@ -82,7 +82,7 @@ class LsCommand(BaseCommand):
|
|||||||
"type": "Type",
|
"type": "Type",
|
||||||
"acl": "Access",
|
"acl": "Access",
|
||||||
"laboratory": "Laboratory",
|
"laboratory": "Laboratory",
|
||||||
"size": "Lock/Size",
|
"size": "Size",
|
||||||
"date": "Date",
|
"date": "Date",
|
||||||
"name": "Name",
|
"name": "Name",
|
||||||
}
|
}
|
||||||
@ -90,11 +90,11 @@ class LsCommand(BaseCommand):
|
|||||||
for key in label.keys():
|
for key in label.keys():
|
||||||
length[key] = len(label[key]) if not context.is_quick else 0
|
length[key] = len(label[key]) if not context.is_quick else 0
|
||||||
for sub_folder in folder.sub_folders:
|
for sub_folder in folder.sub_folders:
|
||||||
sub_laboratory = context.connection.laboratories.find_by_id(sub_folder.lab_id)
|
sub_laboratory = context.connection.laboratories.find_by_id(sub_folder.laboratory_id)
|
||||||
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
|
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
|
||||||
length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
|
length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
|
||||||
length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
|
length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
|
||||||
length["size"] = max(length["size"], len(sub_folder.lock_name))
|
length["size"] = max(length["size"], len(str(folder.size)))
|
||||||
length["date"] = max(length["date"], len(sub_folder.updated_at_name))
|
length["date"] = max(length["date"], len(sub_folder.updated_at_name))
|
||||||
length["name"] = max(length["name"], len(sub_folder.name))
|
length["name"] = max(length["name"], len(sub_folder.name))
|
||||||
for file in folder.files:
|
for file in folder.files:
|
||||||
@ -118,10 +118,11 @@ class LsCommand(BaseCommand):
|
|||||||
print("-" * len(header.expandtabs()))
|
print("-" * len(header.expandtabs()))
|
||||||
|
|
||||||
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
|
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
|
||||||
sub_laboratory_name = cls._laboratory_name(context, sub_folder.lab_id)
|
sub_laboratory_name = cls._laboratory_name(context, sub_folder.laboratory_id)
|
||||||
|
sub_folder_type = "[d]" if sub_folder.lock is False else "[l]"
|
||||||
print(
|
print(
|
||||||
f"{'[d]':{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
|
f"{sub_folder_type:{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
|
||||||
f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.lock_name:{length['size']}}\t"
|
f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.size:{length['size']}}\t"
|
||||||
f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
|
f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
|
||||||
)
|
)
|
||||||
for file in sorted(folder.files, key=lambda x: x.name):
|
for file in sorted(folder.files, key=lambda x: x.name):
|
||||||
@ -134,7 +135,7 @@ class LsCommand(BaseCommand):
|
|||||||
if context.is_recursive:
|
if context.is_recursive:
|
||||||
print("")
|
print("")
|
||||||
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
|
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
|
||||||
folder_api = FolderApi(context.connection)
|
folder_api = FoldersApi(context.connection)
|
||||||
try:
|
try:
|
||||||
if sub_folder.lock:
|
if sub_folder.lock:
|
||||||
folder_api.auth(sub_folder.id, context.password)
|
folder_api.auth(sub_folder.id, context.password)
|
||||||
@ -149,15 +150,16 @@ class LsCommand(BaseCommand):
|
|||||||
"id": folder.id,
|
"id": folder.id,
|
||||||
"pid": folder.pid,
|
"pid": folder.pid,
|
||||||
"name": folder.name,
|
"name": folder.name,
|
||||||
|
"size": folder.size,
|
||||||
"access_level": folder.access_level_name,
|
"access_level": folder.access_level_name,
|
||||||
"lock": folder.lock,
|
"lock": folder.lock,
|
||||||
"laboratory": cls._laboratory_name(context, folder.lab_id),
|
"laboratory": cls._laboratory_name(context, folder.laboratory_id),
|
||||||
"description": folder.description,
|
"description": folder.description,
|
||||||
"created_at": folder.created_at,
|
"created_at": folder.created_at,
|
||||||
"updated_at": folder.updated_at,
|
"updated_at": folder.updated_at,
|
||||||
}
|
}
|
||||||
if isinstance(folder, Folder):
|
if isinstance(folder, Folder):
|
||||||
folder_api = FolderApi(context.connection)
|
folder_api = FoldersApi(context.connection)
|
||||||
data["metadata"] = folder_api.metadata(folder.id)
|
data["metadata"] = folder_api.metadata(folder.id)
|
||||||
if context.is_recursive:
|
if context.is_recursive:
|
||||||
sub_folders: list[dict[str, Any]] = []
|
sub_folders: list[dict[str, Any]] = []
|
||||||
@ -187,7 +189,7 @@ class LsCommand(BaseCommand):
|
|||||||
# "thumbnail": file.thumbnail,
|
# "thumbnail": file.thumbnail,
|
||||||
"description": file.description,
|
"description": file.description,
|
||||||
"metadata": file.metadata,
|
"metadata": file.metadata,
|
||||||
"download_url": f"{context.connection.url}/v2/{file.download_url}",
|
"download_url": f"{context.connection.url}/{file.download_url}",
|
||||||
"created_at": file.created_at,
|
"created_at": file.created_at,
|
||||||
"updated_at": file.updated_at,
|
"updated_at": file.updated_at,
|
||||||
}
|
}
|
||||||
|
@ -2,7 +2,7 @@ import json
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import FolderApi
|
from mdrsclient.api import FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
|
|
||||||
|
|
||||||
@ -26,6 +26,6 @@ class MetadataCommand(BaseCommand):
|
|||||||
connection = cls._create_connection(remote)
|
connection = cls._create_connection(remote)
|
||||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||||
folder = cls._find_folder(connection, laboratory, r_path, password)
|
folder = cls._find_folder(connection, laboratory, r_path, password)
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
metadata = folder_api.metadata(folder.id)
|
metadata = folder_api.metadata(folder.id)
|
||||||
print(json.dumps(metadata, ensure_ascii=False))
|
print(json.dumps(metadata, ensure_ascii=False))
|
||||||
|
@ -3,7 +3,7 @@ from argparse import Namespace
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from unicodedata import normalize
|
from unicodedata import normalize
|
||||||
|
|
||||||
from mdrsclient.api import FolderApi
|
from mdrsclient.api import FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
|
|
||||||
@ -31,5 +31,5 @@ class MkdirCommand(BaseCommand):
|
|||||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
||||||
if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None:
|
if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None:
|
||||||
raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
|
raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
folder_api.create(normalize("NFC", r_basename), parent_folder.id)
|
folder_api.create(normalize("NFC", r_basename), parent_folder.id)
|
||||||
|
@ -3,7 +3,7 @@ from argparse import Namespace
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from unicodedata import normalize
|
from unicodedata import normalize
|
||||||
|
|
||||||
from mdrsclient.api import FileApi, FolderApi
|
from mdrsclient.api import FilesApi, FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
|
|
||||||
@ -52,7 +52,7 @@ class MvCommand(BaseCommand):
|
|||||||
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
|
||||||
if d_sub_folder is not None:
|
if d_sub_folder is not None:
|
||||||
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
|
raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.")
|
||||||
file_api = FileApi(connection)
|
file_api = FilesApi(connection)
|
||||||
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
||||||
file_api.move(s_file, d_parent_folder.id, normalize("NFC", d_basename))
|
file_api.move(s_file, d_parent_folder.id, normalize("NFC", d_basename))
|
||||||
else:
|
else:
|
||||||
@ -67,6 +67,6 @@ class MvCommand(BaseCommand):
|
|||||||
if d_folder.id == s_folder.id:
|
if d_folder.id == s_folder.id:
|
||||||
raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
|
raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.")
|
||||||
raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
|
raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
if s_parent_folder.id != d_parent_folder.id or d_basename != s_basename:
|
||||||
folder_api.move(s_folder, d_parent_folder.id, normalize("NFC", d_basename))
|
folder_api.move(s_folder, d_parent_folder.id, normalize("NFC", d_basename))
|
||||||
|
@ -2,7 +2,7 @@ import os
|
|||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mdrsclient.api import FileApi, FolderApi
|
from mdrsclient.api import FilesApi, FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
|
|
||||||
@ -34,7 +34,7 @@ class RmCommand(BaseCommand):
|
|||||||
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
parent_folder = cls._find_folder(connection, laboratory, r_dirname)
|
||||||
file = parent_folder.find_file(r_basename)
|
file = parent_folder.find_file(r_basename)
|
||||||
if file is not None:
|
if file is not None:
|
||||||
file_api = FileApi(connection)
|
file_api = FilesApi(connection)
|
||||||
file_api.destroy(file)
|
file_api.destroy(file)
|
||||||
else:
|
else:
|
||||||
folder = parent_folder.find_sub_folder(r_basename)
|
folder = parent_folder.find_sub_folder(r_basename)
|
||||||
@ -42,5 +42,5 @@ class RmCommand(BaseCommand):
|
|||||||
raise IllegalArgumentException(f"Cannot remove `{r_path}`: No such file or folder.")
|
raise IllegalArgumentException(f"Cannot remove `{r_path}`: No such file or folder.")
|
||||||
if not is_recursive:
|
if not is_recursive:
|
||||||
raise IllegalArgumentException(f"Cannot remove `{r_path}`: Is a folder.")
|
raise IllegalArgumentException(f"Cannot remove `{r_path}`: Is a folder.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
folder_api.destroy(folder.id, True)
|
folder_api.destroy(folder.id, True)
|
||||||
|
@ -5,7 +5,7 @@ from typing import Any
|
|||||||
|
|
||||||
from pydantic.dataclasses import dataclass
|
from pydantic.dataclasses import dataclass
|
||||||
|
|
||||||
from mdrsclient.api import FileApi, FolderApi
|
from mdrsclient.api import FilesApi, FoldersApi
|
||||||
from mdrsclient.commands.base import BaseCommand
|
from mdrsclient.commands.base import BaseCommand
|
||||||
from mdrsclient.connection import MDRSConnection
|
from mdrsclient.connection import MDRSConnection
|
||||||
from mdrsclient.exceptions import IllegalArgumentException, MDRSException
|
from mdrsclient.exceptions import IllegalArgumentException, MDRSException
|
||||||
@ -26,6 +26,12 @@ class UploadCommand(BaseCommand):
|
|||||||
upload_parser.add_argument(
|
upload_parser.add_argument(
|
||||||
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
||||||
)
|
)
|
||||||
|
upload_parser.add_argument(
|
||||||
|
"-s",
|
||||||
|
"--skip-if-exists",
|
||||||
|
help="skip the upload if file is already uploaded and file size is the same",
|
||||||
|
action="store_true",
|
||||||
|
)
|
||||||
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
||||||
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
||||||
upload_parser.set_defaults(func=cls.func)
|
upload_parser.set_defaults(func=cls.func)
|
||||||
@ -35,12 +41,13 @@ class UploadCommand(BaseCommand):
|
|||||||
local_path = str(args.local_path)
|
local_path = str(args.local_path)
|
||||||
remote_path = str(args.remote_path)
|
remote_path = str(args.remote_path)
|
||||||
is_recursive = bool(args.recursive)
|
is_recursive = bool(args.recursive)
|
||||||
cls.upload(local_path, remote_path, is_recursive)
|
is_skip_if_exists = bool(args.skip_if_exists)
|
||||||
|
cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
|
def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
|
||||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||||
l_path = os.path.realpath(local_path)
|
l_path = os.path.abspath(local_path)
|
||||||
if not os.path.exists(l_path):
|
if not os.path.exists(l_path):
|
||||||
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
||||||
connection = cls._create_connection(remote)
|
connection = cls._create_connection(remote)
|
||||||
@ -50,11 +57,11 @@ class UploadCommand(BaseCommand):
|
|||||||
if os.path.isdir(l_path):
|
if os.path.isdir(l_path):
|
||||||
if not is_recursive:
|
if not is_recursive:
|
||||||
raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
|
raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
|
||||||
folder_api = FolderApi(connection)
|
folder_api = FoldersApi(connection)
|
||||||
folder_map: dict[str, Folder] = {}
|
folder_map: dict[str, Folder] = {}
|
||||||
folder_map[r_path] = folder
|
folder_map[r_path] = folder
|
||||||
l_basename = os.path.basename(l_path)
|
l_basename = os.path.basename(l_path)
|
||||||
for dirpath, _, filenames in os.walk(l_path):
|
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
||||||
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
||||||
d_dirname = os.path.join(r_path, sub)
|
d_dirname = os.path.join(r_path, sub)
|
||||||
d_basename = os.path.basename(d_dirname)
|
d_basename = os.path.basename(d_dirname)
|
||||||
@ -78,23 +85,25 @@ class UploadCommand(BaseCommand):
|
|||||||
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
||||||
else:
|
else:
|
||||||
infos.append(UploadFileInfo(folder, l_path))
|
infos.append(UploadFileInfo(folder, l_path))
|
||||||
cls.__multiple_upload(connection, infos)
|
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
|
def __multiple_upload(
|
||||||
file_api = FileApi(connection)
|
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
|
||||||
|
) -> None:
|
||||||
|
file_api = FilesApi(connection)
|
||||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||||
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
|
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_upload_worker(cls, file_api: FileApi, info: UploadFileInfo) -> None:
|
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||||
basename = os.path.basename(info.path)
|
basename = os.path.basename(info.path)
|
||||||
file = info.folder.find_file(basename)
|
file = info.folder.find_file(basename)
|
||||||
try:
|
try:
|
||||||
if file is None:
|
if file is None:
|
||||||
file_api.create(info.folder.id, info.path)
|
file_api.create(info.folder.id, info.path)
|
||||||
else:
|
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
||||||
file_api.update(file, info.path)
|
file_api.update(file, info.path)
|
||||||
print(os.path.join(info.folder.path, basename))
|
print(os.path.join(info.folder.path, basename))
|
||||||
except MDRSException as e:
|
except MDRSException as e:
|
||||||
print(f"API Error: {e}")
|
print(f"Error: {e}")
|
||||||
|
@ -2,7 +2,7 @@ import configparser
|
|||||||
import os
|
import os
|
||||||
from typing import Final
|
from typing import Final
|
||||||
|
|
||||||
import validators # type: ignore
|
import validators
|
||||||
|
|
||||||
from mdrsclient.exceptions import IllegalArgumentException
|
from mdrsclient.exceptions import IllegalArgumentException
|
||||||
from mdrsclient.settings import CONFIG_DIRNAME
|
from mdrsclient.settings import CONFIG_DIRNAME
|
||||||
@ -41,7 +41,7 @@ class ConfigFile:
|
|||||||
|
|
||||||
@url.setter
|
@url.setter
|
||||||
def url(self, url: str) -> None:
|
def url(self, url: str) -> None:
|
||||||
if not validators.url(url): # type: ignore
|
if not validators.url(url):
|
||||||
raise IllegalArgumentException("malformed URI sequence")
|
raise IllegalArgumentException("malformed URI sequence")
|
||||||
self.__load()
|
self.__load()
|
||||||
if self.__config.has_section(self.remote):
|
if self.__config.has_section(self.remote):
|
||||||
|
@ -1,9 +1,9 @@
|
|||||||
import platform
|
import platform
|
||||||
import threading
|
import threading
|
||||||
from io import BufferedReader
|
|
||||||
from typing import TypedDict
|
from typing import TypedDict
|
||||||
|
|
||||||
from requests import Response, Session
|
from requests import Response, Session
|
||||||
|
from requests_toolbelt.multipart.encoder import MultipartEncoder
|
||||||
|
|
||||||
# Unpack is new in 3.11
|
# Unpack is new in 3.11
|
||||||
from typing_extensions import Unpack
|
from typing_extensions import Unpack
|
||||||
@ -21,14 +21,14 @@ class _KwArgsMDRSConnectionGet(TypedDict, total=False):
|
|||||||
|
|
||||||
class _KwArgsMDRSConnectionPost(TypedDict, total=False):
|
class _KwArgsMDRSConnectionPost(TypedDict, total=False):
|
||||||
params: dict[str, str | int]
|
params: dict[str, str | int]
|
||||||
data: dict[str, str | int]
|
data: dict[str, str | int] | MultipartEncoder
|
||||||
files: dict[str, BufferedReader]
|
headers: dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
class _KwArgsMDRSConnectionPut(TypedDict, total=False):
|
class _KwArgsMDRSConnectionPut(TypedDict, total=False):
|
||||||
params: dict[str, str | int]
|
params: dict[str, str | int]
|
||||||
data: dict[str, str | int]
|
data: dict[str, str | int] | MultipartEncoder
|
||||||
files: dict[str, BufferedReader]
|
headers: dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
class _KwArgsMDRSConnectionDelete(TypedDict, total=False):
|
class _KwArgsMDRSConnectionDelete(TypedDict, total=False):
|
||||||
|
@ -15,7 +15,7 @@ class FolderAccessLevelItem(NamedTuple):
|
|||||||
|
|
||||||
class FolderAccessLevel:
|
class FolderAccessLevel:
|
||||||
# Bit Mask
|
# Bit Mask
|
||||||
# - bit 0: Storage or Repository
|
# - bit 0: Is Private
|
||||||
# - bit 1: Is Public
|
# - bit 1: Is Public
|
||||||
# - bit 2: With Password
|
# - bit 2: With Password
|
||||||
# - bit 3-7: (Reserved)
|
# - bit 3-7: (Reserved)
|
||||||
@ -49,7 +49,8 @@ class FolderSimple:
|
|||||||
name: str
|
name: str
|
||||||
access_level: int
|
access_level: int
|
||||||
lock: bool
|
lock: bool
|
||||||
lab_id: int
|
size: int
|
||||||
|
laboratory_id: int
|
||||||
description: str
|
description: str
|
||||||
created_at: str
|
created_at: str
|
||||||
updated_at: str
|
updated_at: str
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "mdrs-client-python"
|
name = "mdrs-client-python"
|
||||||
version = "1.2.0"
|
version = "1.3.11"
|
||||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@ -13,6 +13,7 @@ classifiers=[
|
|||||||
"Programming Language :: Python :: 3.10",
|
"Programming Language :: Python :: 3.10",
|
||||||
"Programming Language :: Python :: 3.11",
|
"Programming Language :: Python :: 3.11",
|
||||||
"Programming Language :: Python :: 3.12",
|
"Programming Language :: Python :: 3.12",
|
||||||
|
"Programming Language :: Python :: 3.13",
|
||||||
"OSI Approved :: MIT License",
|
"OSI Approved :: MIT License",
|
||||||
"Topic :: Utilities",
|
"Topic :: Utilities",
|
||||||
]
|
]
|
||||||
@ -22,19 +23,20 @@ packages = [
|
|||||||
|
|
||||||
[tool.poetry.dependencies]
|
[tool.poetry.dependencies]
|
||||||
python = "^3.10"
|
python = "^3.10"
|
||||||
requests = "^2.31.0"
|
requests = "^2.32.3"
|
||||||
python-dotenv = "^1.0.0"
|
requests-toolbelt = "^1.0.0"
|
||||||
pydantic = "^2.4.2"
|
python-dotenv = "^1.0.1"
|
||||||
pydantic-settings = "^2.0.3"
|
pydantic = "^2.10.5"
|
||||||
PyJWT = "^2.8.0"
|
pydantic-settings = "^2.7.1"
|
||||||
validators = "^0.22.0"
|
PyJWT = "^2.10.1"
|
||||||
|
validators = "^0.34.0"
|
||||||
|
|
||||||
[tool.poetry.group.dev.dependencies]
|
[tool.poetry.group.dev.dependencies]
|
||||||
black = "^23.9.1"
|
black = "^24.10.0"
|
||||||
flake8 = "^6.1.0"
|
flake8 = "^7.1.1"
|
||||||
Flake8-pyproject = "^1.2.3"
|
Flake8-pyproject = "^1.2.3"
|
||||||
isort = "^5.12.0"
|
isort = "^5.13.2"
|
||||||
pyright = "^1.1.329"
|
pyright = "^1.1.391"
|
||||||
|
|
||||||
[tool.poetry.scripts]
|
[tool.poetry.scripts]
|
||||||
mdrs = 'mdrsclient.__main__:main'
|
mdrs = 'mdrsclient.__main__:main'
|
||||||
|
Loading…
x
Reference in New Issue
Block a user