implemented -s --skip-if-file-exists option for download commnad.

This commit is contained in:
Yoshihiro OKUMURA 2024-09-18 10:56:40 +09:00
parent f2c5a06cb4
commit 24172dc65c
Signed by: orrisroot
GPG Key ID: 470AA444C92904B2

View File

@ -22,6 +22,7 @@ class DownloadFileInfo:
@dataclass @dataclass
class DownloadContext: class DownloadContext:
hasError: bool hasError: bool
isSkipIfExists: bool
files: list[DownloadFileInfo] files: list[DownloadFileInfo]
@ -32,6 +33,12 @@ class DownloadCommand(BaseCommand):
download_parser.add_argument( download_parser.add_argument(
"-r", "--recursive", help="download folders and their contents recursive", action="store_true" "-r", "--recursive", help="download folders and their contents recursive", action="store_true"
) )
download_parser.add_argument(
"-s",
"--skip-if-exists",
help="skip the download if file is already downloaded and file size is the same",
action="store_true",
)
download_parser.add_argument( download_parser.add_argument(
"-e", "--exclude", help="exclude to download path matched file or folders", action="append" "-e", "--exclude", help="exclude to download path matched file or folders", action="append"
) )
@ -45,13 +52,15 @@ class DownloadCommand(BaseCommand):
remote_path = str(args.remote_path) remote_path = str(args.remote_path)
local_path = str(args.local_path) local_path = str(args.local_path)
is_recursive = bool(args.recursive) is_recursive = bool(args.recursive)
is_skip_if_exists = bool(args.skip_if_exists)
password = str(args.password) if args.password else None password = str(args.password) if args.password else None
excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else [] excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
cls.download(remote_path, local_path, is_recursive, password, excludes) cls.download(remote_path, local_path, is_recursive, is_skip_if_exists, password, excludes)
@classmethod @classmethod
def download( def download(
cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None, excludes: list[str] cls, remote_path: str, local_path: str, is_recursive: bool, is_skip_if_exists: bool, password: str | None,
excludes: list[str]
) -> None: ) -> None:
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path) (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
r_path = r_path.rstrip("/") r_path = r_path.rstrip("/")
@ -67,7 +76,7 @@ class DownloadCommand(BaseCommand):
if file is not None: if file is not None:
if cls.__check_excludes(excludes, laboratory, r_parent_folder, file): if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
return return
context = DownloadContext(False, []) context = DownloadContext(False, is_skip_if_exists, [])
l_path = os.path.join(l_dirname, r_basename) l_path = os.path.join(l_dirname, r_basename)
context.files.append(DownloadFileInfo(file, l_path)) context.files.append(DownloadFileInfo(file, l_path))
cls.__multiple_download(connection, context) cls.__multiple_download(connection, context)
@ -79,7 +88,7 @@ class DownloadCommand(BaseCommand):
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.") raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
folder_api = FoldersApi(connection) folder_api = FoldersApi(connection)
cls.__multiple_download_pickup_recursive_files( cls.__multiple_download_pickup_recursive_files(
connection, folder_api, laboratory, folder.id, l_dirname, excludes connection, folder_api, laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
) )
@classmethod @classmethod
@ -91,8 +100,9 @@ class DownloadCommand(BaseCommand):
folder_id: str, folder_id: str,
basedir: str, basedir: str,
excludes: list[str], excludes: list[str],
is_skip_if_exists: bool
) -> None: ) -> None:
context = DownloadContext(False, []) context = DownloadContext(False, is_skip_if_exists, [])
folder = folder_api.retrieve(folder_id) folder = folder_api.retrieve(folder_id)
dirname = os.path.join(basedir, folder.name) dirname = os.path.join(basedir, folder.name)
if cls.__check_excludes(excludes, laboratory, folder, None): if cls.__check_excludes(excludes, laboratory, folder, None):
@ -110,24 +120,27 @@ class DownloadCommand(BaseCommand):
raise UnexpectedException("Some files failed to download.") raise UnexpectedException("Some files failed to download.")
for sub_folder in folder.sub_folders: for sub_folder in folder.sub_folders:
cls.__multiple_download_pickup_recursive_files( cls.__multiple_download_pickup_recursive_files(
connection, folder_api, laboratory, sub_folder.id, dirname, excludes connection, folder_api, laboratory, sub_folder.id, dirname, excludes, is_skip_if_exists
) )
@classmethod @classmethod
def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None: def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
file_api = FilesApi(connection) file_api = FilesApi(connection)
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool: with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
results = pool.map(lambda x: cls.__multiple_download_worker(file_api, x), context.files) results = pool.map(
lambda x: cls.__multiple_download_worker(file_api, x, context.isSkipIfExists), context.files
)
hasError = next(filter(lambda x: x is False, results), None) hasError = next(filter(lambda x: x is False, results), None)
if hasError is not None: if hasError is not None:
context.hasError = True context.hasError = True
@classmethod @classmethod
def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> bool: def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
try: if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
file_api.download(info.file, info.path) try:
except Exception: file_api.download(info.file, info.path)
return False except Exception:
return False
print(info.path) print(info.path)
return True return True