2 Commits

2 changed files with 22 additions and 13 deletions

View File

@ -26,6 +26,12 @@ class UploadCommand(BaseCommand):
upload_parser.add_argument( upload_parser.add_argument(
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true" "-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
) )
upload_parser.add_argument(
"-s",
"--skip-if-exists",
help="skip the upload if file is already uploaded and file size is the same",
action="store_true",
)
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)") upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)") upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
upload_parser.set_defaults(func=cls.func) upload_parser.set_defaults(func=cls.func)
@ -35,10 +41,11 @@ class UploadCommand(BaseCommand):
local_path = str(args.local_path) local_path = str(args.local_path)
remote_path = str(args.remote_path) remote_path = str(args.remote_path)
is_recursive = bool(args.recursive) is_recursive = bool(args.recursive)
cls.upload(local_path, remote_path, is_recursive) is_skip_if_exists = bool(args.skip_if_exists)
cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
@classmethod @classmethod
def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None: def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path) (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
l_path = os.path.abspath(local_path) l_path = os.path.abspath(local_path)
if not os.path.exists(l_path): if not os.path.exists(l_path):
@ -78,22 +85,24 @@ class UploadCommand(BaseCommand):
infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename))) infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
else: else:
infos.append(UploadFileInfo(folder, l_path)) infos.append(UploadFileInfo(folder, l_path))
cls.__multiple_upload(connection, infos) cls.__multiple_upload(connection, infos, is_skip_if_exists)
@classmethod @classmethod
def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None: def __multiple_upload(
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
) -> None:
file_api = FilesApi(connection) file_api = FilesApi(connection)
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool: with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos) pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
@classmethod @classmethod
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo) -> None: def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
basename = os.path.basename(info.path) basename = os.path.basename(info.path)
file = info.folder.find_file(basename) file = info.folder.find_file(basename)
try: try:
if file is None: if file is None:
file_api.create(info.folder.id, info.path) file_api.create(info.folder.id, info.path)
else: elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
file_api.update(file, info.path) file_api.update(file, info.path)
print(os.path.join(info.folder.path, basename)) print(os.path.join(info.folder.path, basename))
except MDRSException as e: except MDRSException as e:

View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "mdrs-client-python" name = "mdrs-client-python"
version = "1.3.2" version = "1.3.3"
description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository." description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"] authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
license = "MIT" license = "MIT"
@ -23,18 +23,18 @@ packages = [
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.10" python = "^3.10"
requests = "^2.31.0" requests = "^2.31.0"
python-dotenv = "^1.0.0" python-dotenv = "^1.0.1"
pydantic = "^2.5.2" pydantic = "^2.6.1"
pydantic-settings = "^2.1.0" pydantic-settings = "^2.1.0"
PyJWT = "^2.8.0" PyJWT = "^2.8.0"
validators = "^0.22.0" validators = "^0.22.0"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
black = "^24.1.1" black = "^24.2.0"
flake8 = "^7.0.0" flake8 = "^7.0.0"
Flake8-pyproject = "^1.2.3" Flake8-pyproject = "^1.2.3"
isort = "^5.13.0" isort = "^5.13.2"
pyright = "^1.1.339" pyright = "^1.1.350"
[tool.poetry.scripts] [tool.poetry.scripts]
mdrs = 'mdrsclient.__main__:main' mdrs = 'mdrsclient.__main__:main'