update version to 1.3.6.

.cspell.json

@@ -1,7 +1,14 @@
 {
   "version": "0.2",
   "language": "en,en-gb",
-  "ignoreWords": ["followlinks", "getframe", "pycache", "pydantic", "UNLCK"],
+  "ignoreWords": [
+    "followlinks",
+    "getframe",
+    "pycache",
+    "pydantic",
+    "toolbelt",
+    "UNLCK"
+  ],
   "words": [
     "chacl",
     "kikan",

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS
+Copyright (c) 2023- Neuroinformatics Unit, RIKEN CBS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -80,6 +80,7 @@ Upload the file or directory
 ```shell
 mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
 mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
+mdrs upload -r -s ./dataset neurodata:/NIU/Repository/TEST/
 ```
 
 ### download
@@ -115,8 +116,8 @@ mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST
 Remove the file or folder
 
 ```shell
-mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs rm neurodata:/NIU/Repository/TEST/sample.dat
-mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset
+mdrs rm -r neurodata:/NIU/Repository/TEST/dataset
 ```
 
 ### chacl
@@ -126,7 +127,7 @@ Change the folder access level
 ```shell
 mdrs chacl private neurodata:/NIU/Repository/Private
 mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
-mdrs chacl pw_open -r -p FOLDER_PASSWORD neurodata:/NIU/Repository/PW_Open
+mdrs chacl pw_open -r -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open
 ```
 
 ### metadata

mdrsclient/VERSION

@@ -1 +1 @@
-1.3.1
+1.3.6

mdrsclient/api/files.py

@@ -1,8 +1,10 @@
+import mimetypes
 import os
 from typing import Any, Final
 
 from pydantic import TypeAdapter
 from pydantic.dataclasses import dataclass
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from mdrsclient.api.base import BaseApi
 from mdrsclient.api.utils import token_check
@@ -17,6 +19,7 @@ class FilesApiCreateResponse:
 
 class FilesApi(BaseApi):
     ENTRYPOINT: Final[str] = "v3/files/"
+    FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
 
     def retrieve(self, id: str) -> File:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
@@ -30,30 +33,43 @@ class FilesApi(BaseApi):
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT
         token_check(self.connection)
-        data: dict[str, str | int] = {"folder_id": folder_id}
+        data: dict[str, str | int] | MultipartEncoder = {}
         try:
             with open(os.path.realpath(path), mode="rb") as fp:
-                response = self.connection.post(url, data=data, files={"file": fp})
+                data = MultipartEncoder(
+                    fields={"folder_id": folder_id, "file": (os.path.basename(path), fp, self._get_mime_type(path))}
+                )
+                response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
                 self._raise_response_error(response)
                 ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
         except OSError:
             raise UnexpectedException(f"Could not open `{path}` file.")
+        except MemoryError:
+            raise UnexpectedException("Out of memory.")
+        except Exception as e:
+            raise UnexpectedException("Unspecified error.") from e
         return ret.id
 
     def update(self, file: File, path: str | None) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + file.id + "/"
         token_check(self.connection)
+        data: dict[str, str | int] | MultipartEncoder = {}
         if path is not None:
             # update file body
             try:
                 with open(os.path.realpath(path), mode="rb") as fp:
-                    response = self.connection.put(url, files={"file": fp})
+                    data = MultipartEncoder(fields={"file": (os.path.basename(path), fp, self._get_mime_type(path))})
+                    response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
             except OSError:
                 raise UnexpectedException(f"Could not open `{path}` file.")
+            except MemoryError:
+                raise UnexpectedException("Out of memory.")
+            except Exception as e:
+                raise UnexpectedException("Unspecified error.") from e
         else:
             # update metadata
-            data: dict[str, str | int] = {"name": file.name, "description": file.description}
+            data = {"name": file.name, "description": file.description}
             response = self.connection.put(url, data=data)
         self._raise_response_error(response)
         return True
@@ -95,6 +111,7 @@ class FilesApi(BaseApi):
     def download(self, file: File, path: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = file.download_url
+        token_check(self.connection)
         response = self.connection.get(url, stream=True)
         self._raise_response_error(response)
         try:
@@ -106,3 +123,9 @@ class FilesApi(BaseApi):
         except PermissionError:
             print(f"Cannot create file `{path}`: Permission denied.")
         return True
+
+    def _get_mime_type(self, path: str) -> str:
+        mt = mimetypes.guess_type(path)
+        if mt:
+            return mt[0] or self.FALLBACK_MIMETYPE
+        return self.FALLBACK_MIMETYPE

mdrsclient/commands/config.py

@@ -25,11 +25,11 @@ class ConfigCommand(BaseCommand):
         update_parser.add_argument("url", help="API entrypoint url of remote host")
         update_parser.set_defaults(func=cls.func_update)
         # config list
-        list_parser = config_parsers.add_parser("list", help="list all the remote hosts")
+        list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
         list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
         list_parser.set_defaults(func=cls.func_list)
         # config delete
-        delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host")
+        delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host", aliases=["remove"])
         delete_parser.add_argument("remote", help="label of remote host")
         delete_parser.set_defaults(func=cls.func_delete)
 

mdrsclient/commands/download.py

@@ -8,7 +8,7 @@ from pydantic.dataclasses import dataclass
 from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
 from mdrsclient.models import File
 from mdrsclient.settings import CONCURRENT
 
@@ -19,6 +19,12 @@ class DownloadFileInfo:
     path: str
 
 
+@dataclass
+class DownloadContext:
+    hasError: bool
+    files: list[DownloadFileInfo]
+
+
 class DownloadCommand(BaseCommand):
     @classmethod
     def register(cls, parsers: Any) -> None:
@@ -52,10 +58,11 @@ class DownloadCommand(BaseCommand):
         laboratory = cls._find_laboratory(connection, laboratory_name)
         r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
         file = r_parent_folder.find_file(r_basename)
-        download_files: list[DownloadFileInfo] = []
         if file is not None:
+            context = DownloadContext(False, [])
             l_path = os.path.join(l_dirname, r_basename)
-            download_files.append(DownloadFileInfo(file, l_path))
+            context.files.append(DownloadFileInfo(file, l_path))
+            cls.__multiple_download(connection, context)
         else:
             folder = r_parent_folder.find_sub_folder(r_basename)
             if folder is None:
@@ -63,13 +70,13 @@ class DownloadCommand(BaseCommand):
             if not is_recursive:
                 raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
             folder_api = FoldersApi(connection)
-            cls.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname)
+            cls.__multiple_download_pickup_recursive_files(connection, folder_api, folder.id, l_dirname)
-        cls.__multiple_download(connection, download_files)
 
     @classmethod
     def __multiple_download_pickup_recursive_files(
-        cls, folder_api: FoldersApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str
+        cls, connection: MDRSConnection, folder_api: FoldersApi, folder_id: str, basedir: str
     ) -> None:
+        context = DownloadContext(False, [])
         folder = folder_api.retrieve(folder_id)
         dirname = os.path.join(basedir, folder.name)
         if not os.path.exists(dirname):
@@ -77,17 +84,27 @@ class DownloadCommand(BaseCommand):
         print(dirname)
         for file in folder.files:
             path = os.path.join(dirname, file.name)
-            infolist.append(DownloadFileInfo(file, path))
+            context.files.append(DownloadFileInfo(file, path))
+        cls.__multiple_download(connection, context)
+        if context.hasError:
+            raise UnexpectedException("Some files failed to download.")
         for sub_folder in folder.sub_folders:
-            cls.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname)
+            cls.__multiple_download_pickup_recursive_files(connection, folder_api, sub_folder.id, dirname)
 
     @classmethod
-    def __multiple_download(cls, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None:
+    def __multiple_download(cls, connection: MDRSConnection, context: DownloadContext) -> None:
         file_api = FilesApi(connection)
         with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_download_worker(file_api, x), infolist)
+            results = pool.map(lambda x: cls.__multiple_download_worker(file_api, x), context.files)
+            hasError = next(filter(lambda x: x is False, results), None)
+            if hasError is not None:
+                context.hasError = True
 
     @classmethod
-    def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> None:
+    def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo) -> bool:
+        try:
             file_api.download(info.file, info.path)
+        except Exception:
+            return False
         print(info.path)
+        return True

mdrsclient/commands/upload.py

@@ -26,6 +26,12 @@ class UploadCommand(BaseCommand):
         upload_parser.add_argument(
             "-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
         )
+        upload_parser.add_argument(
+            "-s",
+            "--skip-if-exists",
+            help="skip the upload if file is already uploaded and file size is the same",
+            action="store_true",
+        )
         upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
         upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
         upload_parser.set_defaults(func=cls.func)
@@ -35,10 +41,11 @@ class UploadCommand(BaseCommand):
         local_path = str(args.local_path)
         remote_path = str(args.remote_path)
         is_recursive = bool(args.recursive)
-        cls.upload(local_path, remote_path, is_recursive)
+        is_skip_if_exists = bool(args.skip_if_exists)
+        cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
 
     @classmethod
-    def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
+    def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
         (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
         l_path = os.path.abspath(local_path)
         if not os.path.exists(l_path):
@@ -78,23 +85,25 @@ class UploadCommand(BaseCommand):
                     infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
         else:
             infos.append(UploadFileInfo(folder, l_path))
-        cls.__multiple_upload(connection, infos)
+        cls.__multiple_upload(connection, infos, is_skip_if_exists)
 
     @classmethod
-    def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
+    def __multiple_upload(
+        cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
+    ) -> None:
         file_api = FilesApi(connection)
         with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
+            pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
 
     @classmethod
-    def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo) -> None:
+    def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
         basename = os.path.basename(info.path)
         file = info.folder.find_file(basename)
         try:
             if file is None:
                 file_api.create(info.folder.id, info.path)
-            else:
+            elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
                 file_api.update(file, info.path)
             print(os.path.join(info.folder.path, basename))
         except MDRSException as e:
-            print(f"API Error: {e}")
+            print(f"Error: {e}")

mdrsclient/connection.py

@@ -1,9 +1,9 @@
 import platform
 import threading
-from io import BufferedReader
 from typing import TypedDict
 
 from requests import Response, Session
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 # Unpack is new in 3.11
 from typing_extensions import Unpack
@@ -21,14 +21,14 @@ class _KwArgsMDRSConnectionGet(TypedDict, total=False):
 
 class _KwArgsMDRSConnectionPost(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
 
 
 class _KwArgsMDRSConnectionPut(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
 
 
 class _KwArgsMDRSConnectionDelete(TypedDict, total=False):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mdrs-client-python"
-version = "1.3.2"
+version = "1.3.6"
 description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
 authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
 license = "MIT"
@@ -22,19 +22,20 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10"
-requests = "^2.31.0"
+requests = "^2.32.3"
-python-dotenv = "^1.0.0"
+requests-toolbelt = "^1.0.0"
-pydantic = "^2.5.2"
+python-dotenv = "^1.0.1"
-pydantic-settings = "^2.1.0"
+pydantic = "^2.8.2"
+pydantic-settings = "^2.3.4"
 PyJWT = "^2.8.0"
 validators = "^0.22.0"
 
 [tool.poetry.group.dev.dependencies]
-black = "^24.1.1"
+black = "^24.2.2"
-flake8 = "^7.0.0"
+flake8 = "^7.1.0"
 Flake8-pyproject = "^1.2.3"
-isort = "^5.13.0"
+isort = "^5.13.2"
-pyright = "^1.1.339"
+pyright = "^1.1.370"
 
 [tool.poetry.scripts]
 mdrs = 'mdrsclient.__main__:main'