update version to 1.3.5.

.cspell.json

@@ -1,7 +1,14 @@
 {
   "version": "0.2",
   "language": "en,en-gb",
-  "ignoreWords": ["getframe", "pycache", "pydantic", "UNLCK"],
+  "ignoreWords": [
+    "followlinks",
+    "getframe",
+    "pycache",
+    "pydantic",
+    "toolbelt",
+    "UNLCK"
+  ],
   "words": [
     "chacl",
     "kikan",

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS
+Copyright (c) 2023- Neuroinformatics Unit, RIKEN CBS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -4,7 +4,7 @@ The mdrs-client-python is python library and a command-line client for up- and d
 
 ## Installing
 
-```
+```shell
 poetry install
 ```
 
@@ -14,141 +14,144 @@ poetry install
 
 Create remote host configuration
 
-```
+```shell
-$ mdrs config create neurodata https://neurodata.riken.jp/api
+mdrs config create neurodata https://neurodata.riken.jp/api
 ```
 
 ### login
 
 Login to remote host
 
-```
+```shell
-$ mdrs login neurodata:
+mdrs login neurodata:
 Username: (enter your login name)
 Password: (enter your password)
+
+mdrs login -u USERNAME -p PASSWORD neurodata:
 ```
 
 ### logout
 
 Logout from remote host
 
-```
+```shell
-$ mdrs logout neurodata:
+mdrs logout neurodata:
 ```
 
 ### whoami
 
 Print current user name
 
-```
+```shell
-$ mdrs whoami neurodata:
+mdrs whoami neurodata:
 ```
 
 ### labs
 
 List all laboratories
 
-```
+```shell
-$ mdrs labs neurodata:
+mdrs labs neurodata:
 ```
 
 ### ls
 
 List the folder contents
 
-```
+```shell
-$ mdrs ls neurodata:/NIU/Repository/
+mdrs ls neurodata:/NIU/Repository/
-$ mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
+mdrs ls -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
-$ mdrs ls -r neurodata:/NIU/Repository/Dataset1/
+mdrs ls -r neurodata:/NIU/Repository/Dataset1/
-$ mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
+mdrs ls -J -r neurodata:/NIU/Repository/Dataset1/
 ```
 
 ### mkdir
 
 Create a new folder
 
-```
+```shell
-$ mdrs mkdir neurodata:/NIU/Repository/TEST
+mdrs mkdir neurodata:/NIU/Repository/TEST
 ```
 
 ### upload
 
 Upload the file or directory
 
-```
+```shell
-$ mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
+mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/
-$ mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
+mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/
+mdrs upload -r -s ./dataset neurodata:/NIU/Repository/TEST/
 ```
 
 ### download
 
 Download the file or folder
 
-```
+```shell
-$ mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
+mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./
-$ mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
+mdrs download -r neurodata:/NIU/Repository/TEST/dataset/ ./
-$ mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
+mdrs download -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.dat ./
 ```
 
 ### mv
 
 Move or rename the file or folder
 
-```
+```shell
-$ mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs mv neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
-$ mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
+mdrs mv neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
 ```
 
 ### cp
 
 Copy the file and folder
 
-```
+```shell
-$ mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs cp neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat
-$ mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
+mdrs cp -r neurodata:/NIU/Repository/TEST/dataset neurodata:/NIU/Repository/TEST2/
 ```
 
 ### rm
 
 Remove the file or folder
 
-```
+```shell
-$ mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat
+mdrs rm neurodata:/NIU/Repository/TEST/sample.dat
-$ mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset
+mdrs rm -r neurodata:/NIU/Repository/TEST/dataset
 ```
 
 ### chacl
 
 Change the folder access level
 
-```
+```shell
-$ mdrs chacl private neurodata:/NIU/Repository/Private
+mdrs chacl private neurodata:/NIU/Repository/Private
-$ mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
+mdrs chacl cbs_open -r neurodata:/NIU/Repository/CBS_Open
-$ mdrs chacl pw_open -r -p FOLDER_PASSWORD neurodata:/NIU/Repository/PW_Open
+mdrs chacl pw_open -r -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open
 ```
 
 ### metadata
 
 Get a folder metadata
 
-```
+```shell
-$ mdrs metadata neurodata:/NIU/Repository/TEST/
+mdrs metadata neurodata:/NIU/Repository/TEST/
-$ mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
+mdrs metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/
 ```
 
 ### file-metadata
 
 Get the file metadata
 
-```
+```shell
-$ mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
+mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat
-$ mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
+mdrs file-metadata -p PW_OPEN_PASSWORD neurodata:/NIU/Repository/PW_Open/Readme.txt
 ```
 
 ### help
 
 Show the help message and exit
 
-```
+```shell
-$ mdrs -h
+mdrs -h
 ```

mdrsclient/VERSION

@@ -1 +1 @@
-1.3.0
+1.3.5

mdrsclient/api/files.py

@@ -1,8 +1,10 @@
+import mimetypes
 import os
 from typing import Any, Final
 
 from pydantic import TypeAdapter
 from pydantic.dataclasses import dataclass
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from mdrsclient.api.base import BaseApi
 from mdrsclient.api.utils import token_check
@@ -17,6 +19,7 @@ class FilesApiCreateResponse:
 
 class FilesApi(BaseApi):
     ENTRYPOINT: Final[str] = "v3/files/"
+    FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
 
     def retrieve(self, id: str) -> File:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
@@ -30,30 +33,43 @@ class FilesApi(BaseApi):
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT
         token_check(self.connection)
-        data: dict[str, str | int] = {"folder_id": folder_id}
+        data: dict[str, str | int] | MultipartEncoder = {}
         try:
-            with open(os.path.relpath(path), mode="rb") as fp:
+            with open(os.path.realpath(path), mode="rb") as fp:
-                response = self.connection.post(url, data=data, files={"file": fp})
+                data = MultipartEncoder(
+                    fields={"folder_id": folder_id, "file": (os.path.basename(path), fp, self._get_mime_type(path))}
+                )
+                response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
                 self._raise_response_error(response)
                 ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
         except OSError:
             raise UnexpectedException(f"Could not open `{path}` file.")
+        except MemoryError:
+            raise UnexpectedException("Out of memory.")
+        except Exception as e:
+            raise UnexpectedException("Unspecified error.") from e
         return ret.id
 
     def update(self, file: File, path: str | None) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + file.id + "/"
         token_check(self.connection)
+        data: dict[str, str | int] | MultipartEncoder = {}
         if path is not None:
             # update file body
             try:
-                with open(os.path.relpath(path), mode="rb") as fp:
+                with open(os.path.realpath(path), mode="rb") as fp:
-                    response = self.connection.put(url, files={"file": fp})
+                    data = MultipartEncoder(fields={"file": (os.path.basename(path), fp, self._get_mime_type(path))})
+                    response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
             except OSError:
                 raise UnexpectedException(f"Could not open `{path}` file.")
+            except MemoryError:
+                raise UnexpectedException("Out of memory.")
+            except Exception as e:
+                raise UnexpectedException("Unspecified error.") from e
         else:
             # update metadata
-            data: dict[str, str | int] = {"name": file.name, "description": file.description}
+            data = {"name": file.name, "description": file.description}
             response = self.connection.put(url, data=data)
         self._raise_response_error(response)
         return True
@@ -95,6 +111,7 @@ class FilesApi(BaseApi):
     def download(self, file: File, path: str) -> bool:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = file.download_url
+        token_check(self.connection)
         response = self.connection.get(url, stream=True)
         self._raise_response_error(response)
         try:
@@ -106,3 +123,9 @@ class FilesApi(BaseApi):
         except PermissionError:
             print(f"Cannot create file `{path}`: Permission denied.")
         return True
+
+    def _get_mime_type(self, path: str) -> str:
+        mt = mimetypes.guess_type(path)
+        if mt:
+            return mt[0] or self.FALLBACK_MIMETYPE
+        return self.FALLBACK_MIMETYPE

mdrsclient/commands/config.py

@@ -25,11 +25,11 @@ class ConfigCommand(BaseCommand):
         update_parser.add_argument("url", help="API entrypoint url of remote host")
         update_parser.set_defaults(func=cls.func_update)
         # config list
-        list_parser = config_parsers.add_parser("list", help="list all the remote hosts")
+        list_parser = config_parsers.add_parser("list", help="list all the remote hosts", aliases=["ls"])
         list_parser.add_argument("-l", "--long", help="show the api url", action="store_true")
         list_parser.set_defaults(func=cls.func_list)
         # config delete
-        delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host")
+        delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host", aliases=["remove"])
         delete_parser.add_argument("remote", help="label of remote host")
         delete_parser.set_defaults(func=cls.func_delete)
 

mdrsclient/commands/login.py

@@ -13,14 +13,16 @@ class LoginCommand(BaseCommand):
     @classmethod
     def register(cls, parsers: Any) -> None:
         login_parser = parsers.add_parser("login", help="login to remote host")
+        login_parser.add_argument("-u", "--username", help="login username")
+        login_parser.add_argument("-p", "--password", help="login password")
         login_parser.add_argument("remote", help="label of remote host")
         login_parser.set_defaults(func=cls.func)
 
     @classmethod
     def func(cls, args: Namespace) -> None:
         remote = str(args.remote)
-        username = input("Username: ").strip()
+        username = str(args.username) if args.password else input("Username: ").strip()
-        password = getpass.getpass("Password: ").strip()
+        password = str(args.password) if args.password else getpass.getpass("Password: ").strip()
         cls.login(remote, username, password)
 
     @classmethod

mdrsclient/commands/upload.py

@@ -26,6 +26,12 @@ class UploadCommand(BaseCommand):
         upload_parser.add_argument(
             "-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
         )
+        upload_parser.add_argument(
+            "-s",
+            "--skip-if-exists",
+            help="skip the upload if file is already uploaded and file size is the same",
+            action="store_true",
+        )
         upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
         upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
         upload_parser.set_defaults(func=cls.func)
@@ -35,12 +41,13 @@ class UploadCommand(BaseCommand):
         local_path = str(args.local_path)
         remote_path = str(args.remote_path)
         is_recursive = bool(args.recursive)
-        cls.upload(local_path, remote_path, is_recursive)
+        is_skip_if_exists = bool(args.skip_if_exists)
+        cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
 
     @classmethod
-    def upload(cls, local_path: str, remote_path: str, is_recursive: bool) -> None:
+    def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
         (remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
-        l_path = local_path
+        l_path = os.path.abspath(local_path)
         if not os.path.exists(l_path):
             raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
         connection = cls._create_connection(remote)
@@ -54,8 +61,9 @@ class UploadCommand(BaseCommand):
             folder_map: dict[str, Folder] = {}
             folder_map[r_path] = folder
             l_basename = os.path.basename(l_path)
-            for dirpath, _, filenames in os.walk(l_path):
+            for dirpath, _, filenames in os.walk(l_path, followlinks=True):
-                d_dirname = os.path.join(r_path, l_basename)
+                sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
+                d_dirname = os.path.join(r_path, sub)
                 d_basename = os.path.basename(d_dirname)
                 # prepare destination parent path
                 d_parent_dirname = os.path.dirname(d_dirname)
@@ -77,23 +85,25 @@ class UploadCommand(BaseCommand):
                     infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
         else:
             infos.append(UploadFileInfo(folder, l_path))
-        cls.__multiple_upload(connection, infos)
+        cls.__multiple_upload(connection, infos, is_skip_if_exists)
 
     @classmethod
-    def __multiple_upload(cls, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None:
+    def __multiple_upload(
+        cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
+    ) -> None:
         file_api = FilesApi(connection)
         with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
-            pool.map(lambda x: cls.__multiple_upload_worker(file_api, x), infos)
+            pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
 
     @classmethod
-    def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo) -> None:
+    def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
         basename = os.path.basename(info.path)
         file = info.folder.find_file(basename)
         try:
             if file is None:
                 file_api.create(info.folder.id, info.path)
-            else:
+            elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
                 file_api.update(file, info.path)
             print(os.path.join(info.folder.path, basename))
         except MDRSException as e:
-            print(f"API Error: {e}")
+            print(f"Error: {e}")

mdrsclient/connection.py

@@ -1,9 +1,9 @@
 import platform
 import threading
-from io import BufferedReader
 from typing import TypedDict
 
 from requests import Response, Session
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 # Unpack is new in 3.11
 from typing_extensions import Unpack
@@ -21,14 +21,14 @@ class _KwArgsMDRSConnectionGet(TypedDict, total=False):
 
 class _KwArgsMDRSConnectionPost(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
 
 
 class _KwArgsMDRSConnectionPut(TypedDict, total=False):
     params: dict[str, str | int]
-    data: dict[str, str | int]
+    data: dict[str, str | int] | MultipartEncoder
-    files: dict[str, BufferedReader]
+    headers: dict[str, str]
 
 
 class _KwArgsMDRSConnectionDelete(TypedDict, total=False):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mdrs-client-python"
-version = "1.3.0"
+version = "1.3.5"
 description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
 authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
 license = "MIT"
@@ -22,19 +22,20 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10"
-requests = "^2.31.0"
+requests = "^2.32.3"
-python-dotenv = "^1.0.0"
+requests-toolbelt = "^1.0.0"
-pydantic = "^2.5.2"
+python-dotenv = "^1.0.1"
-pydantic-settings = "^2.1.0"
+pydantic = "^2.8.2"
+pydantic-settings = "^2.3.4"
 PyJWT = "^2.8.0"
 validators = "^0.22.0"
 
 [tool.poetry.group.dev.dependencies]
-black = "^23.12.0"
+black = "^24.2.2"
-flake8 = "^6.1.0"
+flake8 = "^7.1.0"
 Flake8-pyproject = "^1.2.3"
-isort = "^5.13.0"
+isort = "^5.13.2"
-pyright = "^1.1.339"
+pyright = "^1.1.370"
 
 [tool.poetry.scripts]
 mdrs = 'mdrsclient.__main__:main'