implemented --exclude argument for download subcommand.

README.md

@@ -91,6 +91,7 @@ Download the file or folder
 mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
 mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
 mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
+mdrs download -r  --exclude /NIU/Repository/TEST/dataset/skip neurodata:/NIU/Repository/TEST/dataset/ ./
 ```
 
 ### mv

mdrsclient/VERSION

@@ -1 +1 @@
-1.3.4
+1.3.7

mdrsclient/api/files.py

@@ -47,8 +47,7 @@ class FilesApi(BaseApi):
         except MemoryError:
             raise UnexpectedException("Out of memory.")
         except Exception as e:
-            print(e)
-            raise UnexpectedException(e)
+            raise UnexpectedException("Unspecified error.") from e
         return ret.id
 
     def update(self, file: File, path: str | None) -> bool:
@@ -66,6 +65,8 @@ class FilesApi(BaseApi):
                 raise UnexpectedException(f"Could not open `{path}` file.")
             except MemoryError:
                 raise UnexpectedException("Out of memory.")
+            except Exception as e:
+                raise UnexpectedException("Unspecified error.") from e
         else:
             # update metadata
             data = {"name": file.name, "description": file.description}
@@ -110,6 +111,7 @@ class FilesApi(BaseApi):
     def download(self, file: File, path: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = file.download_url
+        token_check(self.connection)
         response = self.connection.get(url, stream=True)
         self._raise_response_error(response)
         try:

mdrsclient/commands/download.py

@@ -8,8 +8,8 @@ from pydantic.dataclasses import dataclass
 from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import IllegalArgumentException
-from mdrsclient.models import File
+from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
+from mdrsclient.models import File, Folder, Laboratory
 from mdrsclient.settings import CONCURRENT
 
 
@@ -19,6 +19,12 @@ class DownloadFileInfo:
     path: str
 
 
+@dataclass
+class DownloadContext:
+    hasError: bool
+    files: list[DownloadFileInfo]
+
+
 class DownloadCommand(BaseCommand):
     @classmethod
     def register(cls, parsers: Any) -> None:
@@ -26,6 +32,9 @@ class DownloadCommand(BaseCommand):
         download_parser.add_argument(
             "-r", "--recursive", help="download folders and their contents recursive", action="store_true"
         )
+        download_parser.add_argument(
+            "-e", "--exclude", help="exclude to download path matched file or folders", action="append"
+        )
         download_parser.add_argument("-p", "--password", help="password to use when open locked folder")
         download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)")
         download_parser.add_argument("local_path", help="local folder path (/foo/bar/)")
@@ -37,10 +46,13 @@ class DownloadCommand(BaseCommand):
         local_path = str(args.local_path)
         is_recursive = bool(args.recursive)
         password = str(args.password) if args.password else None
-        cls.download(remote_path, local_path, is_recursive, password)
+        excludes = list(map(lambda x: str(x).rstrip("/").lower(), args.exclude)) if args.exclude is not None else []
+        cls.download(remote_path, local_path, is_recursive, password, excludes)
 
     @classmethod
-    def download(cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None) -> None:
+    def download(
+        cls, remote_path: str, local_path: str, is_recursive: bool, password: str | None, excludes: list[str]
+    ) -> None:
         (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
         r_path = r_path.rstrip("/")
         r_dirname = os.path.dirname(r_path)
@@ -52,10 +64,13 @@ class DownloadCommand(BaseCommand):
         laboratory = cls._find_laboratory(connection, laboratory_name)
         r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
         file = r_parent_folder.find_file(r_basename)
-        download_files: list[DownloadFileInfo] = []
         if file is not None:
+            if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
+                return
+            context = DownloadContext(False, [])
             l_path = os.path.join(l_dirname, r_basename)
-            download_files.append(DownloadFileInfo(file, l_path))
+            context.files.append(DownloadFileInfo(file, l_path))
+            cls.__multiple_download(connection, context)
         else:
             folder = r_parent_folder.find_sub_folder(r_basename)
             if folder is None:
@@ -63,31 +78,60 @@ class DownloadCommand(BaseCommand):
             if not is_recursive:
                 raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
             folder_api = FoldersApi(connection)
-            cls.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname)
-        cls.__multiple_download(connection, download_files)
+            cls.__multiple_download_pickup_recursive_files(
+                connection, folder_api, laboratory, folder.id, l_dirname, excludes
+            )
 
     @classmethod
     def __multiple_download_pickup_recursive_files(
-        cls, folder_api: FoldersApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str
+        cls,
+        connection: MDRSConnection,
+        folder_api: FoldersApi,
+        laboratory: Laboratory,
+        folder_id: str,
+        basedir: str,
+        excludes: list[str],
     ) -> None:
+        context = DownloadContext(False, [])
         folder = folder_api.retrieve(folder_id)
         dirname = os.path.join(basedir, folder.name)
+        if cls.__check_excludes(excludes, laboratory, folder, None):
+            return
         if not os.path.exists(dirname):
             os.makedirs(dirname)
         print(dirname)
         for file in folder.files:
+            if cls.__check_excludes(excludes, laboratory, folder, file):
+                continue
             path = os.path.join(dirname, file.name)
-            infolist.append(DownloadFileInfo(file, path))
+            context.files.append(DownloadFileInfo(file, path))
+        cls.__multiple_download(connection, context)
+        if context.hasError:
+            raise UnexpectedException("Some files failed to download.")
         for sub_folder in folder.sub_folders:
-            cls.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname)
+            cls.__multiple_download_pickup_recursive_files(
+                connection, folder_api, laboratory, sub_folder.id, dirname, excludes
+            )
 
     @classmethod
-    def __multiple_download(cls, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None:
+    def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
         file_api = FilesApi(connection)
         with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_download_worker(file_api, x), infolist)
+            results = pool.map(lambda x: cls.__multiple_download_worker(file_api, x), context.files)
+            hasError = next(filter(lambda x: x is False, results), None)
+            if hasError is not None:
+                context.hasError = True
 
     @classmethod
-    def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> None:
+    def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> bool:
+        try:
             file_api.download(info.file, info.path)
+        except Exception:
+            return False
         print(info.path)
+        return True
+
+    @classmethod
+    def __check_excludes(cls, excludes: list[str], laboratory: Laboratory, folder: Folder, file: File | None) -> bool:
+        path = f"/{laboratory.name}{folder.path}{file.name if file is not None else ""}".rstrip("/").lower()
+        return path in excludes

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mdrs-client-python"
-version = "1.3.4"
+version = "1.3.7"
 description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
 authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
 license = "MIT"
@@ -28,14 +28,14 @@ python-dotenv = "^1.0.1"
 pydantic = "^2.8.2"
 pydantic-settings = "^2.3.4"
 PyJWT = "^2.8.0"
-validators = "^0.22.0"
+validators = "^0.33.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.2.2"
 flake8 = "^7.1.0"
 Flake8-pyproject = "^1.2.3"
 isort = "^5.13.2"
-pyright = "^1.1.370"
+pyright = "^1.1.372"
 
 [tool.poetry.scripts]
 mdrs = 'mdrsclient.__main__:main'