Compare commits
No commits in common. "main" and "v1.3.7" have entirely different histories.
@ -16,11 +16,7 @@
|
||||
"mdrsclient",
|
||||
"neurodata",
|
||||
"Neuroinformatics",
|
||||
"orcid",
|
||||
"RIKEN"
|
||||
],
|
||||
"ignorePaths": [
|
||||
".env",
|
||||
"__pycache__"
|
||||
]
|
||||
"ignorePaths": [".env", "__pycache__"]
|
||||
}
|
||||
|
3
.gitignore
vendored
3
.gitignore
vendored
@ -160,5 +160,4 @@ cython_debug/
|
||||
.idea/
|
||||
|
||||
# mdrs-cli
|
||||
.neurodatacli.config
|
||||
poetry.toml
|
||||
.neurodatacli.config
|
15
README.md
15
README.md
@ -60,7 +60,7 @@ List the folder contents
|
||||
|
||||
```shell
|
||||
mdrs ls neurodata:/NIU/Repository/
|
||||
mdrs ls -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
mdrs ls -r neurodata:/NIU/Repository/Dataset1/
|
||||
mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
|
||||
```
|
||||
@ -80,7 +80,7 @@ Upload the file or directory
|
||||
```shell
|
||||
mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
|
||||
mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
|
||||
mdrs upload -r --skip-if-exists ./dataset neurodata:/NIU/Repository/TEST/
|
||||
mdrs upload -r -s ./dataset neurodata:/NIU/Repository/TEST/
|
||||
```
|
||||
|
||||
### download
|
||||
@ -90,9 +90,8 @@ Download the file or folder
|
||||
```shell
|
||||
mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
|
||||
mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -r --skip-if-exists neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
|
||||
mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
|
||||
```
|
||||
|
||||
### mv
|
||||
@ -129,7 +128,7 @@ Change the folder access level
|
||||
```shell
|
||||
mdrs chacl private neurodata:/NIU/Repository/Private
|
||||
mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
|
||||
mdrs chacl pw_open -r -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||
mdrs chacl pw_open -r -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open
|
||||
```
|
||||
|
||||
### metadata
|
||||
@ -138,7 +137,7 @@ Get a folder metadata
|
||||
|
||||
```shell
|
||||
mdrs metadata neurodata:/NIU/Repository/TEST/
|
||||
mdrs metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
|
||||
```
|
||||
|
||||
### file-metadata
|
||||
@ -147,7 +146,7 @@ Get the file metadata
|
||||
|
||||
```shell
|
||||
mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
|
||||
mdrs file-metadata -p SHARING_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||
mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
|
||||
```
|
||||
|
||||
### help
|
||||
|
@ -1 +1 @@
|
||||
1.3.11
|
||||
1.3.7
|
||||
|
@ -1,6 +1,5 @@
|
||||
import argparse
|
||||
import sys
|
||||
from json import JSONDecodeError
|
||||
|
||||
from mdrsclient.commands import (
|
||||
ChaclCommand,
|
||||
@ -53,9 +52,6 @@ def main() -> None:
|
||||
except MDRSException as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(2)
|
||||
except JSONDecodeError:
|
||||
print("Unexpected response returned. Please check the configuration or the server's operational status.")
|
||||
sys.exit(2)
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
|
||||
|
@ -20,10 +20,8 @@ class UsersCurrentResponseLaboratory:
|
||||
class UsersApiCurrentResponse:
|
||||
id: int
|
||||
username: str
|
||||
first_name: str
|
||||
last_name: str
|
||||
full_name: str
|
||||
email: str
|
||||
orcid_id: str
|
||||
laboratories: list[UsersCurrentResponseLaboratory]
|
||||
is_staff: bool
|
||||
is_active: bool
|
||||
|
@ -22,7 +22,6 @@ class DownloadFileInfo:
|
||||
@dataclass
|
||||
class DownloadContext:
|
||||
hasError: bool
|
||||
isSkipIfExists: bool
|
||||
files: list[DownloadFileInfo]
|
||||
|
||||
|
||||
@ -33,12 +32,6 @@ class DownloadCommand(BaseCommand):
|
||||
download_parser.add_argument(
|
||||
"-r", "--recursive", help="download folders and their contents recursive", action="store_true"
|
||||
)
|
||||
download_parser.add_argument(
|
||||
"-s",
|
||||
"--skip-if-exists",
|
||||
help="skip the download if file is already downloaded and file size is the same",
|
||||
action="store_true",
|
||||
)
|
||||
download_parser.add_argument(
|
||||
"-e", "--exclude", help="exclude to download path matched file or folders", action="append"
|
||||
)
|
||||
@ -52,20 +45,13 @@ class DownloadCommand(BaseCommand):
|
||||
remote_path = str(args.remote_path)
|
||||
local_path = str(args.local_path)
|
||||
is_recursive = bool(args.recursive)
|
||||
is_skip_if_exists = bool(args.skip_if_exists)
|
||||
password = str(args.password) if args.password else None
|
||||
excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
|
||||
cls.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
|
||||
cls.download(remote_path, local_path, is_recursive, password, excludes)
|
||||
|
||||
@classmethod
|
||||
def download(
|
||||
cls,
|
||||
remote_path: str,
|
||||
local_path: str,
|
||||
is_recursive: bool,
|
||||
is_skip_if_exists: bool,
|
||||
password: str | None,
|
||||
excludes: list[str],
|
||||
cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None, excludes: list[str]
|
||||
) -> None:
|
||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
@ -81,7 +67,7 @@ class DownloadCommand(BaseCommand):
|
||||
if file is not None:
|
||||
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
|
||||
return
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
context = DownloadContext(False, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
cls.__multiple_download(connection, context)
|
||||
@ -93,7 +79,7 @@ class DownloadCommand(BaseCommand):
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
folder_api = FoldersApi(connection)
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
connection, folder_api, laboratory, folder.id, l_dirname, excludes
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@ -105,9 +91,8 @@ class DownloadCommand(BaseCommand):
|
||||
folder_id: str,
|
||||
basedir: str,
|
||||
excludes: list[str],
|
||||
is_skip_if_exists: bool,
|
||||
) -> None:
|
||||
context = DownloadContext(False, is_skip_if_exists, [])
|
||||
context = DownloadContext(False, [])
|
||||
folder = folder_api.retrieve(folder_id)
|
||||
dirname = os.path.join(basedir, folder.name)
|
||||
if cls.__check_excludes(excludes, laboratory, folder, None):
|
||||
@ -125,34 +110,28 @@ class DownloadCommand(BaseCommand):
|
||||
raise UnexpectedException("Some files failed to download.")
|
||||
for sub_folder in folder.sub_folders:
|
||||
cls.__multiple_download_pickup_recursive_files(
|
||||
connection, folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
|
||||
connection, folder_api, laboratory, sub_folder.id, dirname, excludes
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
|
||||
file_api = FilesApi(connection)
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||
results = pool.map(
|
||||
lambda x: cls.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
|
||||
)
|
||||
results = pool.map(lambda x: cls.__multiple_download_worker(file_api, x), context.files)
|
||||
hasError = next(filter(lambda x: x is False, results), None)
|
||||
if hasError is not None:
|
||||
context.hasError = True
|
||||
|
||||
@classmethod
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
|
||||
if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
|
||||
try:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
print(f"Failed: ${info.path}")
|
||||
if os.path.isfile(info.path):
|
||||
os.remove(info.path)
|
||||
return False
|
||||
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> bool:
|
||||
try:
|
||||
file_api.download(info.file, info.path)
|
||||
except Exception:
|
||||
return False
|
||||
print(info.path)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
|
||||
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ''}".rstrip("/").lower()
|
||||
path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ""}".rstrip("/").lower()
|
||||
return path in excludes
|
||||
|
@ -2,7 +2,7 @@ import configparser
|
||||
import os
|
||||
from typing import Final
|
||||
|
||||
import validators
|
||||
import validators # type: ignore
|
||||
|
||||
from mdrsclient.exceptions import IllegalArgumentException
|
||||
from mdrsclient.settings import CONFIG_DIRNAME
|
||||
@ -41,7 +41,7 @@ class ConfigFile:
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str) -> None:
|
||||
if not validators.url(url):
|
||||
if not validators.url(url): # type: ignore
|
||||
raise IllegalArgumentException("malformed URI sequence")
|
||||
self.__load()
|
||||
if self.__config.has_section(self.remote):
|
||||
|
@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "mdrs-client-python"
|
||||
version = "1.3.11"
|
||||
version = "1.3.7"
|
||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||
license = "MIT"
|
||||
@ -13,7 +13,6 @@ classifiers=[
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"OSI Approved :: MIT License",
|
||||
"Topic :: Utilities",
|
||||
]
|
||||
@ -26,17 +25,17 @@ python = "^3.10"
|
||||
requests = "^2.32.3"
|
||||
requests-toolbelt = "^1.0.0"
|
||||
python-dotenv = "^1.0.1"
|
||||
pydantic = "^2.10.5"
|
||||
pydantic-settings = "^2.7.1"
|
||||
PyJWT = "^2.10.1"
|
||||
validators = "^0.34.0"
|
||||
pydantic = "^2.8.2"
|
||||
pydantic-settings = "^2.3.4"
|
||||
PyJWT = "^2.8.0"
|
||||
validators = "^0.33.0"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^24.10.0"
|
||||
flake8 = "^7.1.1"
|
||||
black = "^24.2.2"
|
||||
flake8 = "^7.1.0"
|
||||
Flake8-pyproject = "^1.2.3"
|
||||
isort = "^5.13.2"
|
||||
pyright = "^1.1.391"
|
||||
pyright = "^1.1.372"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
mdrs = 'mdrsclient.__main__:main'
|
||||
|
Loading…
x
Reference in New Issue
Block a user