From f2c5a06cb4b3e819ea58bf1c99ccd89733a7e4ef Mon Sep 17 00:00:00 2001 From: Yoshihiro OKUMURA Date: Mon, 22 Jul 2024 14:35:34 +0900 Subject: [PATCH] implemented --exclude argument for download subcommand. --- README.md | 1 + mdrsclient/VERSION | 2 +- mdrsclient/commands/download.py | 39 ++++++++++++++++++++++++++++----- pyproject.toml | 6 ++--- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 954813b..211b1b4 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Download the file or folder mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./ mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./ +mdrs download -r --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./ ``` ### mv diff --git a/mdrsclient/VERSION b/mdrsclient/VERSION index 95b25ae..3336003 100644 --- a/mdrsclient/VERSION +++ b/mdrsclient/VERSION @@ -1 +1 @@ -1.3.6 +1.3.7 diff --git a/mdrsclient/commands/download.py b/mdrsclient/commands/download.py index d887630..3aab291 100644 --- a/mdrsclient/commands/download.py +++ b/mdrsclient/commands/download.py @@ -9,7 +9,7 @@ from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException -from mdrsclient.models import File +from mdrsclient.models import File, Folder, Laboratory from mdrsclient.settings import CONCURRENT @@ -32,6 +32,9 @@ class DownloadCommand(BaseCommand): download_parser.add_argument( "-r", "--recursive", help="download folders and their contents recursive", action="store_true" ) + download_parser.add_argument( + "-e", "--exclude", help="exclude to download path matched file or folders", action="append" + ) download_parser.add_argument("-p", "--password", help="password to use when open locked folder") download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)") download_parser.add_argument("local_path", help="local folder path (/foo/bar/)") @@ -43,10 +46,13 @@ class DownloadCommand(BaseCommand): local_path = str(args.local_path) is_recursive = bool(args.recursive) password = str(args.password) if args.password else None - cls.download(remote_path, local_path, is_recursive, password) + excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else [] + cls.download(remote_path, local_path, is_recursive, password, excludes) @classmethod - def download(cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None) -> None: + def download( + cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None, excludes: list[str] + ) -> None: (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path) r_path = r_path.rstrip("/") r_dirname = os.path.dirname(r_path) @@ -59,6 +65,8 @@ class DownloadCommand(BaseCommand): r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password) file = r_parent_folder.find_file(r_basename) if file is not None: + if cls.__check_excludes(excludes, laboratory, r_parent_folder, file): + return context = DownloadContext(False, []) l_path = os.path.join(l_dirname, r_basename) context.files.append(DownloadFileInfo(file, l_path)) @@ -70,26 +78,40 @@ class DownloadCommand(BaseCommand): if not is_recursive: raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.") folder_api = FoldersApi(connection) - cls.__multiple_download_pickup_recursive_files(connection, folder_api, folder.id, l_dirname) + cls.__multiple_download_pickup_recursive_files( + connection, folder_api, laboratory, folder.id, l_dirname, excludes + ) @classmethod def __multiple_download_pickup_recursive_files( - cls, connection: MDRSConnection, folder_api: FoldersApi, folder_id: str, basedir: str + cls, + connection: MDRSConnection, + folder_api: FoldersApi, + laboratory: Laboratory, + folder_id: str, + basedir: str, + excludes: list[str], ) -> None: context = DownloadContext(False, []) folder = folder_api.retrieve(folder_id) dirname = os.path.join(basedir, folder.name) + if cls.__check_excludes(excludes, laboratory, folder, None): + return if not os.path.exists(dirname): os.makedirs(dirname) print(dirname) for file in folder.files: + if cls.__check_excludes(excludes, laboratory, folder, file): + continue path = os.path.join(dirname, file.name) context.files.append(DownloadFileInfo(file, path)) cls.__multiple_download(connection, context) if context.hasError: raise UnexpectedException("Some files failed to download.") for sub_folder in folder.sub_folders: - cls.__multiple_download_pickup_recursive_files(connection, folder_api, sub_folder.id, dirname) + cls.__multiple_download_pickup_recursive_files( + connection, folder_api, laboratory, sub_folder.id, dirname, excludes + ) @classmethod def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None: @@ -108,3 +130,8 @@ class DownloadCommand(BaseCommand): return False print(info.path) return True + + @classmethod + def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool: + path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ""}".rstrip("/").lower() + return path in excludes diff --git a/pyproject.toml b/pyproject.toml index 94db1f8..6f099d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mdrs-client-python" -version = "1.3.6" +version = "1.3.7" description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository." authors = ["Yoshihiro OKUMURA "] license = "MIT" @@ -28,14 +28,14 @@ python-dotenv = "^1.0.1" pydantic = "^2.8.2" pydantic-settings = "^2.3.4" PyJWT = "^2.8.0" -validators = "^0.22.0" +validators = "^0.33.0" [tool.poetry.group.dev.dependencies] black = "^24.2.2" flake8 = "^7.1.0" Flake8-pyproject = "^1.2.3" isort = "^5.13.2" -pyright = "^1.1.370" +pyright = "^1.1.372" [tool.poetry.scripts] mdrs = 'mdrsclient.__main__:main'