Compare commits
No commits in common. "main" and "v1.3.0" have entirely different histories.
15
.cspell.json
15
.cspell.json
@ -1,14 +1,7 @@
|
||||
{
|
||||
"version": "0.2",
|
||||
"language": "en,en-gb",
|
||||
"ignoreWords": [
|
||||
"followlinks",
|
||||
"getframe",
|
||||
"pycache",
|
||||
"pydantic",
|
||||
"toolbelt",
|
||||
"UNLCK"
|
||||
],
|
||||
"ignoreWords": ["getframe", "pycache", "pydantic", "UNLCK"],
|
||||
"words": [
|
||||
"chacl",
|
||||
"kikan",
|
||||
@ -16,11 +9,7 @@
|
||||
"mdrsclient",
|
||||
"neurodata",
|
||||
"Neuroinformatics",
|
||||
"orcid",
|
||||
"RIKEN"
|
||||
],
|
||||
"ignorePaths": [
|
||||
".env",
|
||||
"__pycache__"
|
||||
]
|
||||
"ignorePaths": [".env", "__pycache__"]
|
||||
}
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -161,4 +161,3 @@ cython_debug/
|
||||
|
||||
# mdrs-cli
|
||||
.neurodatacli.config
|
||||
poetry.toml
|
||||
@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023- Neuroinformatics Unit, RIKEN CBS
|
||||
Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
97
README.md
97
README.md
@ -4,7 +4,7 @@ The mdrs-client-python is python library and a command-line client for up- and d
|
||||
|
||||
## Installing
|
||||
|
||||
```shell
|
||||
```
|
||||
poetry install
|
||||
```
|
||||
|
||||
@ -14,146 +14,141 @@ poetry install
|
||||
|
||||
Create remote host configuration
|
||||
|
||||
```shell
|
||||
mdrs config create neurodata https://neurodata.riken.jp/api
|
||||
```
|
||||
$ mdrs config create neurodata https://neurodata.riken.jp/api
|
||||
```
|
||||
|
||||
### login
|
||||
|
||||
Login to remote host
|
||||
|
||||
```shell
|
||||
mdrs login neurodata:
|
||||
```
|
||||
$ mdrs login neurodata:
|
||||
Username: (enter your login name)
|
||||
Password: (enter your password)
|
||||
|
||||
mdrs login -u USERNAME -p PASSWORD neurodata:
|
||||
```
|
||||
|
||||
### logout
|
||||
|
||||
Logout from remote host
|
||||
|
||||
```shell
|
||||
mdrs logout neurodata:
|
||||
```
|
||||
$ mdrs logout neurodata:
|
||||
```
|
||||
|
||||
### whoami
|
||||
|
||||
Print current user name
|
||||
|
||||
```shell
|
||||
mdrs whoami neurodata:
|
||||
```
|
||||
$ mdrs whoami neurodata:
|
||||
```
|
||||
|
||||
### labs
|
||||
|
||||
List all laboratories
|
||||
|
||||
```shell
|
||||
mdrs labs neurodata:
|
||||
```
|
||||
$ mdrs labs neurodata:
|
||||
```
|
||||
|
||||
### ls
|
||||
|
||||
List the folder contents
|
||||
|
||||
```shell
|
||||
mdrs ls neurodata:/NIU/Repository/
|
||||
mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
||||
mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
||||
```
|
||||
$ mdrs ls neurodata:/NIU/Repository/
|
||||
$ mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
$ mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
||||
$ mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
||||
```
|
||||
|
||||
### mkdir
|
||||
|
||||
Create a new folder
|
||||
|
||||
```shell
|
||||
mdrs mkdir neurodata:/NIU/Repository/TEST
|
||||
```
|
||||
$ mdrs mkdir neurodata:/NIU/Repository/TEST
|
||||
```
|
||||
|
||||
### upload
|
||||
|
||||
Upload the file or directory
|
||||
|
||||
```shell
|
||||
mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
|
||||
mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
|
||||
mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
|
||||
```
|
||||
$ mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
|
||||
$ mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
|
||||
```
|
||||
|
||||
### download
|
||||
|
||||
Download the file or folder
|
||||
|
||||
```shell
|
||||
mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
||||
mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
```
|
||||
$ mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
||||
$ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
$ mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||
```
|
||||
|
||||
### mv
|
||||
|
||||
Move or rename the file or folder
|
||||
|
||||
```shell
|
||||
mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||
mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||
```
|
||||
$ mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||
$ mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||
```
|
||||
|
||||
### cp
|
||||
|
||||
Copy the file and folder
|
||||
|
||||
```shell
|
||||
mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||
mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||
```
|
||||
$ mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||
$ mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
|
||||
```
|
||||
|
||||
### rm
|
||||
|
||||
Remove the file or folder
|
||||
|
||||
```shell
|
||||
mdrs rm neurodata:/NIU/Repository/TEST/sample.dat
|
||||
mdrs rm -r neurodata:/NIU/Repository/TEST/dataset
|
||||
```
|
||||
$ mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat
|
||||
$ mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset
|
||||
```
|
||||
|
||||
### chacl
|
||||
|
||||
Change the folder access level
|
||||
|
||||
```shell
|
||||
mdrs chacl private neurodata:/NIU/Repository/Private
|
||||
mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
|
||||
mdrs chacl pw_open -r -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||
```
|
||||
$ mdrs chacl private neurodata:/NIU/Repository/Private
|
||||
$ mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
|
||||
$ mdrs chacl pw_open -r -p FOLDER_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||
```
|
||||
|
||||
### metadata
|
||||
|
||||
Get a folder metadata
|
||||
|
||||
```shell
|
||||
mdrs metadata neurodata:/NIU/Repository/TEST/
|
||||
mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
```
|
||||
$ mdrs metadata neurodata:/NIU/Repository/TEST/
|
||||
$ mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
```
|
||||
|
||||
### file-metadata
|
||||
|
||||
Get the file metadata
|
||||
|
||||
```shell
|
||||
mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
||||
mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||
```
|
||||
$ mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
||||
$ mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||
```
|
||||
|
||||
### help
|
||||
|
||||
Show the help message and exit
|
||||
|
||||
```shell
|
||||
mdrs -h
|
||||
```
|
||||
$ mdrs -h
|
||||
```
|
||||
|
||||
@ -1 +1 @@
|
||||
1.3.11
|
||||
1.3.0
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
import argparse
|
||||
import sys
|
||||
from json import JSONDecodeError
|
||||
|
||||
from mdrsclient.commands import (
|
||||
ChaclCommand,
|
||||
@ -53,9 +52,6 @@ def main() -> None:
|
||||
except MDRSException as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(2)
|
||||
except JSONDecodeError:
|
||||
print("Unexpected response returned. Please check the configuration or the server's operational status.")
|
||||
sys.exit(2)
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
import mimetypes
|
||||
import os
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic.dataclasses import dataclass
|
||||
from requests_toolbelt.multipart.encoder import MultipartEncoder
|
||||
|
||||
from mdrsclient.api.base import BaseApi
|
||||
from mdrsclient.api.utils import token_check
|
||||
@ -19,7 +17,6 @@ class FilesApiCreateResponse:
|
||||
|
||||
class FilesApi(BaseApi):
|
||||
ENTRYPOINT: Final[str] = "v3/files/"
|
||||
FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
|
||||
|
||||
def retrieve(self, id: str) -> File:
|
||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||
@ -33,43 +30,30 @@ class FilesApi(BaseApi):
|
||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||
url = self.ENTRYPOINT
|
||||
token_check(self.connection)
|
||||
data: dict[str, str | int] | MultipartEncoder = {}
|
||||
data: dict[str, str | int] = {"folder_id": folder_id}
|
||||
try:
|
||||
with open(os.path.realpath(path), mode="rb") as fp:
|
||||
data = MultipartEncoder(
|
||||
fields={"folder_id": folder_id, "file": (os.path.basename(path), fp, self._get_mime_type(path))}
|
||||
)
|
||||
response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
|
||||
with open(os.path.relpath(path), mode="rb") as fp:
|
||||
response = self.connection.post(url, data=data, files={"file": fp})
|
||||
self._raise_response_error(response)
|
||||
ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
|
||||
except OSError:
|
||||
raise UnexpectedException(f"Could not open `{path}` file.")
|
||||
except MemoryError:
|
||||
raise UnexpectedException("Out of memory.")
|
||||
except Exception as e:
|
||||
raise UnexpectedException("Unspecified error.") from e
|
||||
return ret.id
|
||||
|
||||
def update(self, file: File, path: str | None) -> bool:
|
||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||
url = self.ENTRYPOINT + file.id + "/"
|
||||
token_check(self.connection)
|
||||
data: dict[str, str | int] | MultipartEncoder = {}
|
||||
if path is not None:
|
||||
# update file body
|
||||
try:
|
||||
with open(os.path.realpath(path), mode="rb") as fp:
|
||||
data = MultipartEncoder(fields={"file": (os.path.basename(path), fp, self._get_mime_type(path))})
|
||||
response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
|
||||
with open(os.path.relpath(path), mode="rb") as fp:
|
||||
response = self.connection.put(url, files={"file": fp})
|
||||
except OSError:
|
||||
raise UnexpectedException(f"Could not open `{path}` file.")
|
||||
except MemoryError:
|
||||
raise UnexpectedException("Out of memory.")
|
||||
except Exception as e:
|
||||
raise UnexpectedException("Unspecified error.") from e
|
||||
else:
|
||||
# update metadata
|
||||
data = {"name": file.name, "description": file.description}
|
||||
data: dict[str, str | int] = {"name": file.name, "description": file.description}
|
||||
response = self.connection.put(url, data=data)
|
||||
self._raise_response_error(response)
|
||||
return True
|
||||
@ -111,7 +95,6 @@ class FilesApi(BaseApi):
|
||||
def download(self, file: File, path: str) -> bool:
|
||||
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
||||
url = file.download_url
|
||||
token_check(self.connection)
|
||||
response = self.connection.get(url, stream=True)
|
||||
self._raise_response_error(response)
|
||||
try:
|
||||
@ -123,9 +106,3 @@ class FilesApi(BaseApi):
|
||||
except PermissionError:
|
||||
print(f"Cannot create file `{path}`: Permission denied.")
|
||||
return True
|
||||
|
||||
def _get_mime_type(self, path: str) -> str:
|
||||
mt = mimetypes.guess_type(path)
|
||||
if mt:
|
||||
return mt[0] or self.FALLBACK_MIMETYPE
|
||||
return self.FALLBACK_MIMETYPE
|
||||
|
||||
@ -20,10 +20,8 @@ class UsersCurrentResponseLaboratory:
|
||||
class UsersApiCurrentResponse:
|
||||
id: int
|
||||
username: str
|
||||
first_name: str
|
||||
last_name: str
|
||||
full_name: str
|
||||
email: str
|
||||
orcid_id: str
|
||||
laboratories: list[UsersCurrentResponseLaboratory]
|
||||
is_staff: bool
|
||||
is_active: bool
|
||||
|
||||
@ -25,11 +25,11 @@ class ConfigCommand(BaseCommand):
|
||||
update_parser.add_argument("url", help="API entrypoint url of remote host")
|
||||
update_parser.set_defaults(func=cls.func_update)
|
||||
# config list
|
||||
list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
|
||||
list_parser = config_parsers.add_parser("list", help="list all the remote hosts")
|
||||
list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
|
||||
list_parser.set_defaults(func=cls.func_list)
|
||||
# config delete
|
||||
delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host", aliases=["remove"])
|
||||
delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host")
|
||||
delete_parser.add_argument("remote", help="label of remote host")
|
||||
delete_parser.set_defaults(func=cls.func_delete)
|
||||
|
||||
|
||||
@ -8,8 +8,8 @@ from pydantic.dataclasses import dataclass
|
||||
from mdrsclient.api import FilesApi, FoldersApi
|
||||
from mdrsclient.commands.base import BaseCommand
|
||||
from mdrsclient.connection import MDRSConnection
|
||||
from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, Laboratory
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.models import File
|
||||
from mdrsclient.settings import CONCURRENT
|
||||
|
||||
|
||||
@ -19,13 +19,6 @@ class DownloadFileInfo:
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
hasError: bool
|
||||
isSkipIfExists: bool
|
||||
files: list[DownloadFileInfo]
|
||||
|
||||
|
||||
class DownloadCommand(BaseCommand):
|
||||
@classmethod
|
||||
def register(cls, parsers: Any) -> None:
|
||||
@ -33,15 +26,6 @@ class DownloadCommand(BaseCommand):
|
||||
download_parser.add_argument(
|
||||
"-r", "--recursive", help="download folders and their contents recursive", action="store_true"
|
||||
)
|
||||
download_parser.add_argument(
|
||||
"-s",
|
||||
"--skip-if-exists",
|
||||
help="skip the download if file is already downloaded and file size is the same",
|
||||
action="store_true",
|
||||
)
|
||||
download_parser.add_argument(
|
||||
"-e", "--exclude", help="exclude to download path matched file or folders", action="append"
|
||||
)
|
||||
download_parser.add_argument("-p", "--password", help="password to use when open locked folder")
|
||||
download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)")
|
||||
download_parser.add_argument("local_path", help="local folder path (/foo/bar/)")
|
||||
@ -52,21 +36,11 @@ class DownloadCommand(BaseCommand):
|
||||
remote_path = str(args.remote_path)
|
||||
local_path = str(args.local_path)
|
||||
is_recursive = bool(args.recursive)
|
||||
is_skip_if_exists = bool(args.skip_if_exists)
|
||||
password = str(args.password) if args.password else None
|
||||
excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
|
||||
cls.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
|
||||
cls.download(remote_path, local_path, is_recursive, password)
|
||||
|
||||
@classmethod
|
||||
def download(
|
||||
cls,
|
||||
remote_path: str,
|
||||
local_path: str,
|
||||
is_recursive: bool,
|
||||
is_skip_if_exists: bool,
|
||||
password: str | None,
|
||||
excludes: list[str],
|
||||
) -> None:
|
||||
def download(cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
@ -78,13 +52,10 @@ class DownloadCommand(BaseCommand):
|
||||
laboratory = cls._find_laboratory(connection, laboratory_name)
|
||||
r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
|
||||
file = r_parent_folder.find_file(r_basename)
|
||||
download_files: list[DownloadFileInfo] = []
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
download_files.append(DownloadFileInfo(file, l_path))
|
||||
else:
|
||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder is None:
|
||||
@ -92,67 +63,31 @@ class DownloadCommand(BaseCommand):
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
folder_api = FoldersApi(connection)
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
cls.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname)
|
||||
cls.__multiple_download(connection, download_files)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_pickup_recursive_files(
|
||||
cls,
|
||||
connection: MDRSConnection,
|
||||
folder_api: FoldersApi,
|
||||
laboratory: Laboratory,
|
||||
folder_id: str,
|
||||
basedir: str,
|
||||
excludes: list[str],
|
||||
is_skip_if_exists: bool,
|
||||
cls, folder_api: FoldersApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str
|
||||
) -> None:
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
folder = folder_api.retrieve(folder_id)
|
||||
dirname = os.path.join(basedir, folder.name)
|
||||
if cls.__check_excludes(excludes, laboratory, folder, None):
|
||||
return
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
print(dirname)
|
||||
for file in folder.files:
|
||||
if cls.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
path = os.path.join(dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, path))
|
||||
cls.__multiple_download(connection, context)
|
||||
if context.hasError:
|
||||
raise UnexpectedException("Some files failed to download.")
|
||||
infolist.append(DownloadFileInfo(file, path))
|
||||
for sub_folder in folder.sub_folders:
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
cls.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
|
||||
def __multiple_download(cls, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None:
|
||||
file_api = FilesApi(connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
results = pool.map(
|
||||
lambda x: cls.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
|
||||
)
|
||||
hasError = next(filter(lambda x: x is False, results), None)
|
||||
if hasError is not None:
|
||||
context.hasError = True
|
||||
pool.map(lambda x: cls.__multiple_download_worker(file_api, x), infolist)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||
if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
|
||||
try:
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> None:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
print(f"Failed: ${info.path}")
|
||||
if os.path.isfile(info.path):
|
||||
os.remove(info.path)
|
||||
return False
|
||||
print(info.path)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
|
||||
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
|
||||
return path in excludes
|
||||
|
||||
@ -13,16 +13,14 @@ class LoginCommand(BaseCommand):
|
||||
@classmethod
|
||||
def register(cls, parsers: Any) -> None:
|
||||
login_parser = parsers.add_parser("login", help="login to remote host")
|
||||
login_parser.add_argument("-u", "--username", help="login username")
|
||||
login_parser.add_argument("-p", "--password", help="login password")
|
||||
login_parser.add_argument("remote", help="label of remote host")
|
||||
login_parser.set_defaults(func=cls.func)
|
||||
|
||||
@classmethod
|
||||
def func(cls, args: Namespace) -> None:
|
||||
remote = str(args.remote)
|
||||
username = str(args.username) if args.password else input("Username: ").strip()
|
||||
password = str(args.password) if args.password else getpass.getpass("Password: ").strip()
|
||||
username = input("Username: ").strip()
|
||||
password = getpass.getpass("Password: ").strip()
|
||||
cls.login(remote, username, password)
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -26,12 +26,6 @@ class UploadCommand(BaseCommand):
|
||||
upload_parser.add_argument(
|
||||
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
||||
)
|
||||
upload_parser.add_argument(
|
||||
"-s",
|
||||
"--skip-if-exists",
|
||||
help="skip the upload if file is already uploaded and file size is the same",
|
||||
action="store_true",
|
||||
)
|
||||
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
||||
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
||||
upload_parser.set_defaults(func=cls.func)
|
||||
@ -41,13 +35,12 @@ class UploadCommand(BaseCommand):
|
||||
local_path = str(args.local_path)
|
||||
remote_path = str(args.remote_path)
|
||||
is_recursive = bool(args.recursive)
|
||||
is_skip_if_exists = bool(args.skip_if_exists)
|
||||
cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||
cls.upload(local_path, remote_path, is_recursive)
|
||||
|
||||
@classmethod
|
||||
def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
|
||||
def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
l_path = os.path.abspath(local_path)
|
||||
l_path = local_path
|
||||
if not os.path.exists(l_path):
|
||||
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
||||
connection = cls._create_connection(remote)
|
||||
@ -61,9 +54,8 @@ class UploadCommand(BaseCommand):
|
||||
folder_map: dict[str, Folder] = {}
|
||||
folder_map[r_path] = folder
|
||||
l_basename = os.path.basename(l_path)
|
||||
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
||||
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
||||
d_dirname = os.path.join(r_path, sub)
|
||||
for dirpath, _, filenames in os.walk(l_path):
|
||||
d_dirname = os.path.join(r_path, l_basename)
|
||||
d_basename = os.path.basename(d_dirname)
|
||||
# prepare destination parent path
|
||||
d_parent_dirname = os.path.dirname(d_dirname)
|
||||
@ -85,25 +77,23 @@ class UploadCommand(BaseCommand):
|
||||
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
||||
else:
|
||||
infos.append(UploadFileInfo(folder, l_path))
|
||||
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
||||
cls.__multiple_upload(connection, infos)
|
||||
|
||||
@classmethod
|
||||
def __multiple_upload(
|
||||
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
|
||||
) -> None:
|
||||
def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
|
||||
file_api = FilesApi(connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
||||
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
|
||||
|
||||
@classmethod
|
||||
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo) -> None:
|
||||
basename = os.path.basename(info.path)
|
||||
file = info.folder.find_file(basename)
|
||||
try:
|
||||
if file is None:
|
||||
file_api.create(info.folder.id, info.path)
|
||||
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
||||
else:
|
||||
file_api.update(file, info.path)
|
||||
print(os.path.join(info.folder.path, basename))
|
||||
except MDRSException as e:
|
||||
print(f"Error: {e}")
|
||||
print(f"API Error: {e}")
|
||||
|
||||
@ -2,7 +2,7 @@ import configparser
|
||||
import os
|
||||
from typing import Final
|
||||
|
||||
import validators
|
||||
import validators # type: ignore
|
||||
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.settings import CONFIG_DIRNAME
|
||||
@ -41,7 +41,7 @@ class ConfigFile:
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str) -> None:
|
||||
if not validators.url(url):
|
||||
if not validators.url(url): # type: ignore
|
||||
raise IllegalArgumentException("malformed URI sequence")
|
||||
self.__load()
|
||||
if self.__config.has_section(self.remote):
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import platform
|
||||
import threading
|
||||
from io import BufferedReader
|
||||
from typing import TypedDict
|
||||
|
||||
from requests import Response, Session
|
||||
from requests_toolbelt.multipart.encoder import MultipartEncoder
|
||||
|
||||
# Unpack is new in 3.11
|
||||
from typing_extensions import Unpack
|
||||
@ -21,14 +21,14 @@ class _KwArgsMDRSConnectionGet(TypedDict, total=False):
|
||||
|
||||
class _KwArgsMDRSConnectionPost(TypedDict, total=False):
|
||||
params: dict[str, str | int]
|
||||
data: dict[str, str | int] | MultipartEncoder
|
||||
headers: dict[str, str]
|
||||
data: dict[str, str | int]
|
||||
files: dict[str, BufferedReader]
|
||||
|
||||
|
||||
class _KwArgsMDRSConnectionPut(TypedDict, total=False):
|
||||
params: dict[str, str | int]
|
||||
data: dict[str, str | int] | MultipartEncoder
|
||||
headers: dict[str, str]
|
||||
data: dict[str, str | int]
|
||||
files: dict[str, BufferedReader]
|
||||
|
||||
|
||||
class _KwArgsMDRSConnectionDelete(TypedDict, total=False):
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "mdrs-client-python"
|
||||
version = "1.3.11"
|
||||
version = "1.3.0"
|
||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||
license = "MIT"
|
||||
@ -13,7 +13,6 @@ classifiers=[
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"OSI Approved :: MIT License",
|
||||
"Topic :: Utilities",
|
||||
]
|
||||
@ -23,20 +22,19 @@ packages = [
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.10"
|
||||
requests = "^2.32.3"
|
||||
requests-toolbelt = "^1.0.0"
|
||||
python-dotenv = "^1.0.1"
|
||||
pydantic = "^2.10.5"
|
||||
pydantic-settings = "^2.7.1"
|
||||
PyJWT = "^2.10.1"
|
||||
validators = "^0.34.0"
|
||||
requests = "^2.31.0"
|
||||
python-dotenv = "^1.0.0"
|
||||
pydantic = "^2.5.2"
|
||||
pydantic-settings = "^2.1.0"
|
||||
PyJWT = "^2.8.0"
|
||||
validators = "^0.22.0"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^24.10.0"
|
||||
flake8 = "^7.1.1"
|
||||
black = "^23.12.0"
|
||||
flake8 = "^6.1.0"
|
||||
Flake8-pyproject = "^1.2.3"
|
||||
isort = "^5.13.2"
|
||||
pyright = "^1.1.391"
|
||||
isort = "^5.13.0"
|
||||
pyright = "^1.1.339"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
mdrs = 'mdrsclient.__main__:main'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user