fix for file.list api pagination

.cspell.json

@@ -16,7 +16,11 @@
     "mdrsclient",
     "neurodata",
     "Neuroinformatics",
+    "orcid",
     "RIKEN"
   ],
-  "ignorePaths": [".env", "__pycache__"]
+  "ignorePaths": [
+    ".env",
+    "__pycache__"
+  ]
 }

mdrsclient/VERSION

@@ -1 +1 @@
-1.3.10
+1.3.12

mdrsclient/api/files.py

@@ -17,10 +17,26 @@ class FilesApiCreateResponse:
     id: str
 
 
+@dataclass(frozen=True)
+class FilesApiListResponse:
+    count: int
+    next: str | None
+    previous: str | None
+    results: list[File]
+
+
 class FilesApi(BaseApi):
     ENTRYPOINT: Final[str] = "v3/files/"
     FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
 
+    def list(self, folder_id: str, page_num: int) -> FilesApiListResponse:
+        url = self.ENTRYPOINT
+        token_check(self.connection)
+        params: dict[str, str | int] = {"folder_id": folder_id, "page": page_num}
+        response = self.connection.get(url, params=params)
+        self._raise_response_error(response)
+        return TypeAdapter(FilesApiListResponse).validate_python(response.json())
+
     def retrieve(self, id: str) -> File:
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = self.ENTRYPOINT + id + "/"

mdrsclient/api/users.py

@@ -20,8 +20,10 @@ class UsersCurrentResponseLaboratory:
 class UsersApiCurrentResponse:
     id: int
     username: str
-    full_name: str
+    first_name: str
+    last_name: str
     email: str
+    orcid_id: str
     laboratories: list[UsersCurrentResponseLaboratory]
     is_staff: bool
     is_active: bool

mdrsclient/commands/base.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from typing import Any
 from unicodedata import normalize
 
-from mdrsclient.api import FoldersApi, LaboratoriesApi
+from mdrsclient.api import FilesApi, FoldersApi, LaboratoriesApi
 from mdrsclient.config import ConfigFile
 from mdrsclient.connection import MDRSConnection
 from mdrsclient.exceptions import (
@@ -12,7 +12,8 @@ from mdrsclient.exceptions import (
     UnauthorizedException,
     UnexpectedException,
 )
-from mdrsclient.models import Folder, Laboratory
+from mdrsclient.models import File, Folder, Laboratory
+from mdrsclient.utils import page_num_from_url
 
 
 class BaseCommand(ABC):
@@ -52,6 +53,19 @@ class BaseCommand(ABC):
             folder_api.auth(folders[0].id, password)
         return folder_api.retrieve(folders[0].id)
 
+    @classmethod
+    def _find_files(cls, connection: MDRSConnection, folder_id: str) -> list[File]:
+        files_api = FilesApi(connection)
+        page = 1
+        results_file = []
+        while page:
+            result = files_api.list(folder_id, page)
+            results_file.extend(result.results)
+            page = 0
+            if result.next:
+                page = page_num_from_url(result.next)
+        return results_file
+
     @classmethod
     def _parse_remote_host(cls, path: str) -> str:
         path_array = path.split(":")

mdrsclient/commands/cp.py

@@ -6,6 +6,7 @@ from unicodedata import normalize
 from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models.file import find_file
 
 
 class CpCommand(BaseCommand):
@@ -46,11 +47,13 @@ class CpCommand(BaseCommand):
         connection = cls._create_connection(s_remote)
         laboratory = cls._find_laboratory(connection, s_laboratory_name)
         s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
+        s_parent_files = cls._find_files(connection, s_parent_folder.id)
         d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
-        s_file = s_parent_folder.find_file(s_basename)
+        d_parent_files = cls._find_files(connection, d_parent_folder.id)
+        s_file = find_file(s_parent_files, s_basename)
         if s_file is not None:
             # source is file
-            d_file = d_parent_folder.find_file(d_basename)
+            d_file = find_file(d_parent_files, d_basename)
             if d_file is not None:
                 raise IllegalArgumentException(f"File `{d_basename}` already exists.")
             d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
@@ -66,7 +69,7 @@ class CpCommand(BaseCommand):
             # source is folder
             if not is_recursive:
                 raise IllegalArgumentException(f"Cannot copy `{s_path}`: Is a folder.")
-            if d_parent_folder.find_file(d_basename) is not None:
+            if find_file(d_parent_files, d_basename) is not None:
                 raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
             d_folder = d_parent_folder.find_sub_folder(d_basename)
             if d_folder is not None:

mdrsclient/commands/download.py

@@ -10,6 +10,7 @@ from mdrsclient.commands.base import BaseCommand
 from mdrsclient.connection import MDRSConnection
 from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException
 from mdrsclient.models import File, Folder, Laboratory
+from mdrsclient.models.file import find_file
 from mdrsclient.settings import CONCURRENT
 
 
@@ -77,7 +78,8 @@ class DownloadCommand(BaseCommand):
             raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
         laboratory = cls._find_laboratory(connection, laboratory_name)
         r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password)
-        file = r_parent_folder.find_file(r_basename)
+        r_parent_files = cls._find_files(connection, r_parent_folder.id)
+        file = find_file(r_parent_files, r_basename)
         if file is not None:
             if cls.__check_excludes(excludes, laboratory, r_parent_folder, file):
                 return
@@ -109,13 +111,14 @@ class DownloadCommand(BaseCommand):
     ) -> None:
         context = DownloadContext(False, is_skip_if_exists, [])
         folder = folder_api.retrieve(folder_id)
+        files = cls._find_files(connection, folder.id)
         dirname = os.path.join(basedir, folder.name)
         if cls.__check_excludes(excludes, laboratory, folder, None):
             return
         if not os.path.exists(dirname):
             os.makedirs(dirname)
         print(dirname)
-        for file in folder.files:
+        for file in files:
             if cls.__check_excludes(excludes, laboratory, folder, file):
                 continue
             path = os.path.join(dirname, file.name)
@@ -141,7 +144,7 @@ class DownloadCommand(BaseCommand):
 
     @classmethod
     def __multiple_download_worker(cls, file_api: FilesApi, info: DownloadFileInfo, is_skip_if_exists: bool) -> bool:
-        if not is_skip_if_exists or os.path.exists(info.path) and info.file.size != os.path.getsize(info.path):
+        if not is_skip_if_exists or not os.path.exists(info.path) or info.file.size != os.path.getsize(info.path):
             try:
                 file_api.download(info.file, info.path)
             except Exception:

mdrsclient/commands/file_metadata.py

@@ -6,6 +6,7 @@ from typing import Any
 from mdrsclient.api import FilesApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models.file import find_file
 
 
 class FileMetadataCommand(BaseCommand):
@@ -31,7 +32,8 @@ class FileMetadataCommand(BaseCommand):
         connection = cls._create_connection(remote)
         laboratory = cls._find_laboratory(connection, laboratory_name)
         folder = cls._find_folder(connection, laboratory, r_dirname, password)
-        file = folder.find_file(r_basename)
+        files = cls._find_files(connection, folder.id)
+        file = find_file(files, r_basename)
         if file is None:
             raise IllegalArgumentException(f"File `{r_basename}` not found.")
         file_api = FilesApi(connection)

mdrsclient/commands/ls.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from pydantic.dataclasses import dataclass
 
-from mdrsclient.api import FoldersApi
+from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.connection import MDRSConnection
 from mdrsclient.exceptions import UnauthorizedException
@@ -67,17 +67,18 @@ class LsCommand(BaseCommand):
             is_recursive,
         )
         folder = cls._find_folder(connection, laboratory, r_path, password)
+        files = cls._find_files(connection, folder.id)
         if context.is_json:
-            cls._ls_json(context, folder)
+            cls._ls_json(context, folder, files)
         else:
-            cls._ls_plain(context, folder)
+            cls._ls_plain(context, folder, files)
 
     @classmethod
-    def _ls_json(cls, context: LsCommandContext, folder: Folder) -> None:
-        print(json.dumps(cls._folder2dict(context, folder), ensure_ascii=False))
+    def _ls_json(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
+        print(json.dumps(cls._folder2dict(context, folder, files), ensure_ascii=False))
 
     @classmethod
-    def _ls_plain(cls, context: LsCommandContext, folder: Folder) -> None:
+    def _ls_plain(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None:
         label = {
             "type": "Type",
             "acl": "Access",
@@ -97,7 +98,7 @@ class LsCommand(BaseCommand):
             length["size"] = max(length["size"], len(str(folder.size)))
             length["date"] = max(length["date"], len(sub_folder.updated_at_name))
             length["name"] = max(length["name"], len(sub_folder.name))
-        for file in folder.files:
+        for file in files:
             length["size"] = max(length["size"], len(str(file.size)))
             length["date"] = max(length["date"], len(file.updated_at_name))
             length["name"] = max(length["name"], len(file.name))
@@ -111,7 +112,7 @@ class LsCommand(BaseCommand):
 
         if context.is_recursive:
             print(f"{context.prefix}{folder.path}:")
-            print(f"total {sum(f.size for f in folder.files)}")
+            print(f"total {sum(f.size for f in files)}")
 
         if not context.is_quick:
             print(header)
@@ -125,7 +126,7 @@ class LsCommand(BaseCommand):
                 f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.size:{length['size']}}\t"
                 f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
             )
-        for file in sorted(folder.files, key=lambda x: x.name):
+        for file in sorted(files, key=lambda x: x.name):
             print(
                 f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t"
                 f"{context.laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t"
@@ -140,12 +141,15 @@ class LsCommand(BaseCommand):
                     if sub_folder.lock:
                         folder_api.auth(sub_folder.id, context.password)
                     folder = folder_api.retrieve(sub_folder.id)
-                    cls._ls_plain(context, folder)
+                    files = cls._find_files(context.connection, sub_folder.id)
+                    cls._ls_plain(context, folder, files)
                 except UnauthorizedException:
                     pass
 
     @classmethod
-    def _folder2dict(cls, context: LsCommandContext, folder: Folder | FolderSimple) -> dict[str, Any]:
+    def _folder2dict(
+        cls, context: LsCommandContext, folder: Folder | FolderSimple, files: list[File]
+    ) -> dict[str, Any]:
         data: dict[str, Any] = {
             "id": folder.id,
             "pid": folder.pid,
@@ -168,15 +172,16 @@ class LsCommand(BaseCommand):
                         if sub_folder.lock:
                             folder_api.auth(sub_folder.id, context.password)
                         folder2 = folder_api.retrieve(sub_folder.id)
-                        sub_folders.append(cls._folder2dict(context, folder2))
+                        files2 = cls._find_files(context.connection, sub_folder.id)
+                        sub_folders.append(cls._folder2dict(context, folder2, files2))
                     except UnauthorizedException:
                         pass
                 data["sub_folders"] = sub_folders
             else:
                 data["sub_folders"] = list(
-                    map(lambda x: cls._folder2dict(context, x), sorted(folder.sub_folders, key=lambda x: x.name))
+                    map(lambda x: cls._folder2dict(context, x, []), sorted(folder.sub_folders, key=lambda x: x.name))
                 )
-            data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(folder.files, key=lambda x: x.name)))
+            data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(files, key=lambda x: x.name)))
         return data
 
     @classmethod

mdrsclient/commands/mkdir.py

@@ -6,6 +6,7 @@ from unicodedata import normalize
 from mdrsclient.api import FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models.file import find_file
 
 
 class MkdirCommand(BaseCommand):
@@ -29,7 +30,8 @@ class MkdirCommand(BaseCommand):
         connection = cls._create_connection(remote)
         laboratory = cls._find_laboratory(connection, laboratory_name)
         parent_folder = cls._find_folder(connection, laboratory, r_dirname)
-        if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None:
+        files = cls._find_files(connection, parent_folder.id)
+        if parent_folder.find_sub_folder(r_basename) is not None or find_file(files, r_basename) is not None:
             raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.")
         folder_api = FoldersApi(connection)
         folder_api.create(normalize("NFC", r_basename), parent_folder.id)

mdrsclient/commands/mv.py

@@ -6,6 +6,7 @@ from unicodedata import normalize
 from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models.file import find_file
 
 
 class MvCommand(BaseCommand):
@@ -42,11 +43,13 @@ class MvCommand(BaseCommand):
         connection = cls._create_connection(s_remote)
         laboratory = cls._find_laboratory(connection, s_laboratory_name)
         s_parent_folder = cls._find_folder(connection, laboratory, s_dirname)
+        s_parent_files = cls._find_files(connection, s_parent_folder.id)
         d_parent_folder = cls._find_folder(connection, laboratory, d_dirname)
-        s_file = s_parent_folder.find_file(s_basename)
+        d_parent_files = cls._find_files(connection, d_parent_folder.id)
+        s_file = find_file(s_parent_files, s_basename)
         if s_file is not None:
             # source is file
-            d_file = d_parent_folder.find_file(d_basename)
+            d_file = find_file(d_parent_files, d_basename)
             if d_file is not None:
                 raise IllegalArgumentException(f"File `{d_basename}` already exists.")
             d_sub_folder = d_parent_folder.find_sub_folder(d_basename)
@@ -60,7 +63,7 @@ class MvCommand(BaseCommand):
             if s_folder is None:
                 raise IllegalArgumentException(f"File or folder `{s_basename}` not found.")
             # source is folder
-            if d_parent_folder.find_file(d_basename) is not None:
+            if find_file(d_parent_files, d_basename) is not None:
                 raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.")
             d_folder = d_parent_folder.find_sub_folder(d_basename)
             if d_folder is not None:

mdrsclient/commands/rm.py

@@ -5,6 +5,7 @@ from typing import Any
 from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.models.file import find_file
 
 
 class RmCommand(BaseCommand):
@@ -32,7 +33,8 @@ class RmCommand(BaseCommand):
         connection = cls._create_connection(remote)
         laboratory = cls._find_laboratory(connection, laboratory_name)
         parent_folder = cls._find_folder(connection, laboratory, r_dirname)
-        file = parent_folder.find_file(r_basename)
+        parent_files = cls._find_files(connection, parent_folder.id)
+        file = find_file(parent_files, r_basename)
         if file is not None:
             file_api = FilesApi(connection)
             file_api.destroy(file)

mdrsclient/commands/upload.py

@@ -9,13 +9,15 @@ from mdrsclient.api import FilesApi, FoldersApi
 from mdrsclient.commands.base import BaseCommand
 from mdrsclient.connection import MDRSConnection
 from mdrsclient.exceptions import IllegalArgumentException, MDRSException
-from mdrsclient.models import Folder
+from mdrsclient.models import File, Folder
+from mdrsclient.models.file import find_file
 from mdrsclient.settings import CONCURRENT
 
 
 @dataclass(frozen=True)
 class UploadFileInfo:
     folder: Folder
+    files: list[File]
     path: str
 
 
@@ -53,6 +55,7 @@ class UploadCommand(BaseCommand):
         connection = cls._create_connection(remote)
         laboratory = cls._find_laboratory(connection, laboratory_name)
         folder = cls._find_folder(connection, laboratory, r_path)
+        files = cls._find_files(connection, folder.id)
         infos: list[UploadFileInfo] = []
         if os.path.isdir(l_path):
             if not is_recursive:
@@ -60,6 +63,8 @@ class UploadCommand(BaseCommand):
             folder_api = FoldersApi(connection)
             folder_map: dict[str, Folder] = {}
             folder_map[r_path] = folder
+            files_map: dict[str, list[File]] = {}
+            files_map[r_path] = files
             l_basename = os.path.basename(l_path)
             for dirpath, _, filenames in os.walk(l_path, followlinks=True):
                 sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
@@ -68,7 +73,10 @@ class UploadCommand(BaseCommand):
                 # prepare destination parent path
                 d_parent_dirname = os.path.dirname(d_dirname)
                 if folder_map.get(d_parent_dirname) is None:
-                    folder_map[d_parent_dirname] = cls._find_folder(connection, laboratory, d_parent_dirname)
+                    parent_folder = cls._find_folder(connection, laboratory, d_parent_dirname)
+                    folder_map[d_parent_dirname] = parent_folder
+                    parent_files = cls._find_files(connection, parent_folder.id)
+                    files_map[d_parent_dirname] = parent_files
                 # prepare destination path
                 if folder_map.get(d_dirname) is None:
                     d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
@@ -78,13 +86,16 @@ class UploadCommand(BaseCommand):
                         d_folder_id = d_folder.id
                     print(d_dirname)
                     folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
+                    files_map[d_dirname] = cls._find_files(connection, d_folder_id)
                     if d_folder is None:
                         folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
                 # register upload file list
                 for filename in filenames:
-                    infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename)))
+                    infos.append(
+                        UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
+                    )
         else:
-            infos.append(UploadFileInfo(folder, l_path))
+            infos.append(UploadFileInfo(folder, files, l_path))
         cls.__multiple_upload(connection, infos, is_skip_if_exists)
 
     @classmethod
@@ -98,7 +109,7 @@ class UploadCommand(BaseCommand):
     @classmethod
     def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
         basename = os.path.basename(info.path)
-        file = info.folder.find_file(basename)
+        file = find_file(info.files, basename)
         try:
             if file is None:
                 file_api.create(info.folder.id, info.path)

mdrsclient/models/file.py

@@ -1,4 +1,5 @@
 from typing import Any
+from unicodedata import normalize
 
 from pydantic.dataclasses import dataclass
 
@@ -25,3 +26,8 @@ class File:
     @property
     def updated_at_name(self) -> str:
         return iso8601_to_user_friendly(self.updated_at)
+
+
+def find_file(files: list[File], name: str) -> File | None:
+    _name = normalize("NFC", name).lower()
+    return next((x for x in files if x.name.lower() == _name), None)

mdrsclient/models/folder.py

@@ -3,7 +3,6 @@ from unicodedata import normalize
 
 from pydantic.dataclasses import dataclass
 
-from mdrsclient.models.file import File
 from mdrsclient.models.utils import iso8601_to_user_friendly
 
 
@@ -78,13 +77,8 @@ class FolderSimple:
 class Folder(FolderSimple):
     metadata: list[dict[str, Any]]
     sub_folders: list[FolderSimple]
-    files: list[File]
     path: str
 
     def find_sub_folder(self, name: str) -> FolderSimple | None:
         _name = normalize("NFC", name).lower()
         return next((x for x in self.sub_folders if x.name.lower() == _name), None)
-
-    def find_file(self, name: str) -> File | None:
-        _name = normalize("NFC", name).lower()
-        return next((x for x in self.files if x.name.lower() == _name), None)

mdrsclient/utils.py

@@ -1,5 +1,6 @@
 import os
 from typing import IO, Any
+from urllib.parse import parse_qs, urlparse
 
 if os.name == "nt":
     import msvcrt
@@ -21,3 +22,10 @@ class FileLock:
             msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
         elif os.name == "posix":
             fcntl.flock(file.fileno(), fcntl.LOCK_UN)
+
+
+def page_num_from_url(url: str) -> int | None:
+    parsed_url = urlparse(url)
+    params = parse_qs(parsed_url.query)
+    page = params.get("page", [None])[0]
+    return int(page) if page is not None else None

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mdrs-client-python"
-version = "1.3.10"
+version = "1.3.12"
 description = "The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository."
 authors = ["Yoshihiro OKUMURA <yoshihiro.okumura@riken.jp>"]
 license = "MIT"
@@ -25,18 +25,18 @@ packages = [
 python = "^3.10"
 requests = "^2.32.3"
 requests-toolbelt = "^1.0.0"
-python-dotenv = "^1.0.1"
-pydantic = "^2.10.4"
-pydantic-settings = "^2.7.0"
+python-dotenv = "^1.1.0"
+pydantic = "^2.11.4"
+pydantic-settings = "^2.9.1"
 PyJWT = "^2.10.1"
 validators = "^0.34.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"
-flake8 = "^7.1.1"
+flake8 = "^7.2.0"
 Flake8-pyproject = "^1.2.3"
 isort = "^5.13.2"
-pyright = "^1.1.386"
+pyright = "^1.1.400"
 
 [tool.poetry.scripts]
 mdrs = 'mdrsclient.__main__:main'