Compare commits
2 Commits
Author | SHA1 | Date | |
---|---|---|---|
55265e69a4
|
|||
dbfc68c396
|
@ -26,6 +26,12 @@ class UploadCommand(BaseCommand):
|
|||||||
upload_parser.add_argument(
|
upload_parser.add_argument(
|
||||||
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
||||||
)
|
)
|
||||||
|
upload_parser.add_argument(
|
||||||
|
"-s",
|
||||||
|
"--skip-if-exists",
|
||||||
|
help="skip the upload if file is already uploaded and file size is the same",
|
||||||
|
action="store_true",
|
||||||
|
)
|
||||||
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
||||||
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
||||||
upload_parser.set_defaults(func=cls.func)
|
upload_parser.set_defaults(func=cls.func)
|
||||||
@ -35,10 +41,11 @@ class UploadCommand(BaseCommand):
|
|||||||
local_path = str(args.local_path)
|
local_path = str(args.local_path)
|
||||||
remote_path = str(args.remote_path)
|
remote_path = str(args.remote_path)
|
||||||
is_recursive = bool(args.recursive)
|
is_recursive = bool(args.recursive)
|
||||||
cls.upload(local_path, remote_path, is_recursive)
|
is_skip_if_exists = bool(args.skip_if_exists)
|
||||||
|
cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
|
def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
|
||||||
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
||||||
l_path = os.path.abspath(local_path)
|
l_path = os.path.abspath(local_path)
|
||||||
if not os.path.exists(l_path):
|
if not os.path.exists(l_path):
|
||||||
@ -78,22 +85,24 @@ class UploadCommand(BaseCommand):
|
|||||||
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
|
||||||
else:
|
else:
|
||||||
infos.append(UploadFileInfo(folder, l_path))
|
infos.append(UploadFileInfo(folder, l_path))
|
||||||
cls.__multiple_upload(connection, infos)
|
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
|
def __multiple_upload(
|
||||||
|
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
|
||||||
|
) -> None:
|
||||||
file_api = FilesApi(connection)
|
file_api = FilesApi(connection)
|
||||||
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
||||||
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
|
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo) -> None:
|
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
||||||
basename = os.path.basename(info.path)
|
basename = os.path.basename(info.path)
|
||||||
file = info.folder.find_file(basename)
|
file = info.folder.find_file(basename)
|
||||||
try:
|
try:
|
||||||
if file is None:
|
if file is None:
|
||||||
file_api.create(info.folder.id, info.path)
|
file_api.create(info.folder.id, info.path)
|
||||||
else:
|
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
||||||
file_api.update(file, info.path)
|
file_api.update(file, info.path)
|
||||||
print(os.path.join(info.folder.path, basename))
|
print(os.path.join(info.folder.path, basename))
|
||||||
except MDRSException as e:
|
except MDRSException as e:
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "mdrs-client-python"
|
name = "mdrs-client-python"
|
||||||
version = "1.3.2"
|
version = "1.3.3"
|
||||||
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
|
||||||
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@ -23,18 +23,18 @@ packages = [
|
|||||||
[tool.poetry.dependencies]
|
[tool.poetry.dependencies]
|
||||||
python = "^3.10"
|
python = "^3.10"
|
||||||
requests = "^2.31.0"
|
requests = "^2.31.0"
|
||||||
python-dotenv = "^1.0.0"
|
python-dotenv = "^1.0.1"
|
||||||
pydantic = "^2.5.2"
|
pydantic = "^2.6.1"
|
||||||
pydantic-settings = "^2.1.0"
|
pydantic-settings = "^2.1.0"
|
||||||
PyJWT = "^2.8.0"
|
PyJWT = "^2.8.0"
|
||||||
validators = "^0.22.0"
|
validators = "^0.22.0"
|
||||||
|
|
||||||
[tool.poetry.group.dev.dependencies]
|
[tool.poetry.group.dev.dependencies]
|
||||||
black = "^24.1.1"
|
black = "^24.2.0"
|
||||||
flake8 = "^7.0.0"
|
flake8 = "^7.0.0"
|
||||||
Flake8-pyproject = "^1.2.3"
|
Flake8-pyproject = "^1.2.3"
|
||||||
isort = "^5.13.0"
|
isort = "^5.13.2"
|
||||||
pyright = "^1.1.339"
|
pyright = "^1.1.350"
|
||||||
|
|
||||||
[tool.poetry.scripts]
|
[tool.poetry.scripts]
|
||||||
mdrs = 'mdrsclient.__main__:main'
|
mdrs = 'mdrsclient.__main__:main'
|
||||||
|
Reference in New Issue
Block a user