From 95f22ea5f9a026eae380d3a45ebf323d06dd1b48 Mon Sep 17 00:00:00 2001 From: serizawa Date: Thu, 26 Jun 2025 17:13:06 +0900 Subject: [PATCH] fix for file.list api pagination --- mdrsclient/api/files.py | 16 ++++++++++++++ mdrsclient/commands/base.py | 18 +++++++++++++-- mdrsclient/commands/cp.py | 9 +++++--- mdrsclient/commands/download.py | 7 ++++-- mdrsclient/commands/file_metadata.py | 4 +++- mdrsclient/commands/ls.py | 33 ++++++++++++++++------------ mdrsclient/commands/mkdir.py | 4 +++- mdrsclient/commands/mv.py | 9 +++++--- mdrsclient/commands/rm.py | 4 +++- mdrsclient/commands/upload.py | 21 +++++++++++++----- mdrsclient/models/file.py | 6 +++++ mdrsclient/models/folder.py | 6 ----- mdrsclient/utils.py | 8 +++++++ 13 files changed, 107 insertions(+), 38 deletions(-) diff --git a/mdrsclient/api/files.py b/mdrsclient/api/files.py index 05e0456..4e75e9e 100644 --- a/mdrsclient/api/files.py +++ b/mdrsclient/api/files.py @@ -17,10 +17,26 @@ class FilesApiCreateResponse: id: str +@dataclass(frozen=True) +class FilesApiListResponse: + count: int + next: str | None + previous: str | None + results: list[File] + + class FilesApi(BaseApi): ENTRYPOINT: Final[str] = "v3/files/" FALLBACK_MIMETYPE: Final[str] = "application/octet-stream" + def list(self, folder_id: str, page_num: int) -> FilesApiListResponse: + url = self.ENTRYPOINT + token_check(self.connection) + params: dict[str, str | int] = {"folder_id": folder_id, "page": page_num} + response = self.connection.get(url, params=params) + self._raise_response_error(response) + return TypeAdapter(FilesApiListResponse).validate_python(response.json()) + def retrieve(self, id: str) -> File: # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) url = self.ENTRYPOINT + id + "/" diff --git a/mdrsclient/commands/base.py b/mdrsclient/commands/base.py index eda67d1..2234678 100644 --- a/mdrsclient/commands/base.py +++ b/mdrsclient/commands/base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import Any from unicodedata import normalize -from mdrsclient.api import FoldersApi, LaboratoriesApi +from mdrsclient.api import FilesApi, FoldersApi, LaboratoriesApi from mdrsclient.config import ConfigFile from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import ( @@ -12,7 +12,8 @@ from mdrsclient.exceptions import ( UnauthorizedException, UnexpectedException, ) -from mdrsclient.models import Folder, Laboratory +from mdrsclient.models import File, Folder, Laboratory +from mdrsclient.utils import page_num_from_url class BaseCommand(ABC): @@ -52,6 +53,19 @@ class BaseCommand(ABC): folder_api.auth(folders[0].id, password) return folder_api.retrieve(folders[0].id) + @classmethod + def _find_files(cls, connection: MDRSConnection, folder_id: str) -> list[File]: + files_api = FilesApi(connection) + page = 1 + results_file = [] + while page: + result = files_api.list(folder_id, page) + results_file.extend(result.results) + page = 0 + if result.next: + page = page_num_from_url(result.next) + return results_file + @classmethod def _parse_remote_host(cls, path: str) -> str: path_array = path.split(":") diff --git a/mdrsclient/commands/cp.py b/mdrsclient/commands/cp.py index 78360a8..cfabb71 100644 --- a/mdrsclient/commands/cp.py +++ b/mdrsclient/commands/cp.py @@ -6,6 +6,7 @@ from unicodedata import normalize from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models.file import find_file class CpCommand(BaseCommand): @@ -46,11 +47,13 @@ class CpCommand(BaseCommand): connection = cls._create_connection(s_remote) laboratory = cls._find_laboratory(connection, s_laboratory_name) s_parent_folder = cls._find_folder(connection, laboratory, s_dirname) + s_parent_files = cls._find_files(connection, s_parent_folder.id) d_parent_folder = cls._find_folder(connection, laboratory, d_dirname) - s_file = s_parent_folder.find_file(s_basename) + d_parent_files = cls._find_files(connection, d_parent_folder.id) + s_file = find_file(s_parent_files, s_basename) if s_file is not None: # source is file - d_file = d_parent_folder.find_file(d_basename) + d_file = find_file(d_parent_files, d_basename) if d_file is not None: raise IllegalArgumentException(f"File `{d_basename}` already exists.") d_sub_folder = d_parent_folder.find_sub_folder(d_basename) @@ -66,7 +69,7 @@ class CpCommand(BaseCommand): # source is folder if not is_recursive: raise IllegalArgumentException(f"Cannot copy `{s_path}`: Is a folder.") - if d_parent_folder.find_file(d_basename) is not None: + if find_file(d_parent_files, d_basename) is not None: raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.") d_folder = d_parent_folder.find_sub_folder(d_basename) if d_folder is not None: diff --git a/mdrsclient/commands/download.py b/mdrsclient/commands/download.py index 5e262f0..ad26010 100644 --- a/mdrsclient/commands/download.py +++ b/mdrsclient/commands/download.py @@ -10,6 +10,7 @@ from mdrsclient.commands.base import BaseCommand from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import IllegalArgumentException, UnexpectedException from mdrsclient.models import File, Folder, Laboratory +from mdrsclient.models.file import find_file from mdrsclient.settings import CONCURRENT @@ -77,7 +78,8 @@ class DownloadCommand(BaseCommand): raise IllegalArgumentException(f"Local directory `{local_path}` not found.") laboratory = cls._find_laboratory(connection, laboratory_name) r_parent_folder = cls._find_folder(connection, laboratory, r_dirname, password) - file = r_parent_folder.find_file(r_basename) + r_parent_files = cls._find_files(connection, r_parent_folder.id) + file = find_file(r_parent_files, r_basename) if file is not None: if cls.__check_excludes(excludes, laboratory, r_parent_folder, file): return @@ -109,13 +111,14 @@ class DownloadCommand(BaseCommand): ) -> None: context = DownloadContext(False, is_skip_if_exists, []) folder = folder_api.retrieve(folder_id) + files = cls._find_files(connection, folder.id) dirname = os.path.join(basedir, folder.name) if cls.__check_excludes(excludes, laboratory, folder, None): return if not os.path.exists(dirname): os.makedirs(dirname) print(dirname) - for file in folder.files: + for file in files: if cls.__check_excludes(excludes, laboratory, folder, file): continue path = os.path.join(dirname, file.name) diff --git a/mdrsclient/commands/file_metadata.py b/mdrsclient/commands/file_metadata.py index 1101d17..6121a7a 100644 --- a/mdrsclient/commands/file_metadata.py +++ b/mdrsclient/commands/file_metadata.py @@ -6,6 +6,7 @@ from typing import Any from mdrsclient.api import FilesApi from mdrsclient.commands.base import BaseCommand from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models.file import find_file class FileMetadataCommand(BaseCommand): @@ -31,7 +32,8 @@ class FileMetadataCommand(BaseCommand): connection = cls._create_connection(remote) laboratory = cls._find_laboratory(connection, laboratory_name) folder = cls._find_folder(connection, laboratory, r_dirname, password) - file = folder.find_file(r_basename) + files = cls._find_files(connection, folder.id) + file = find_file(files, r_basename) if file is None: raise IllegalArgumentException(f"File `{r_basename}` not found.") file_api = FilesApi(connection) diff --git a/mdrsclient/commands/ls.py b/mdrsclient/commands/ls.py index d9d2dfa..715a39c 100644 --- a/mdrsclient/commands/ls.py +++ b/mdrsclient/commands/ls.py @@ -4,7 +4,7 @@ from typing import Any from pydantic.dataclasses import dataclass -from mdrsclient.api import FoldersApi +from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import UnauthorizedException @@ -67,17 +67,18 @@ class LsCommand(BaseCommand): is_recursive, ) folder = cls._find_folder(connection, laboratory, r_path, password) + files = cls._find_files(connection, folder.id) if context.is_json: - cls._ls_json(context, folder) + cls._ls_json(context, folder, files) else: - cls._ls_plain(context, folder) + cls._ls_plain(context, folder, files) @classmethod - def _ls_json(cls, context: LsCommandContext, folder: Folder) -> None: - print(json.dumps(cls._folder2dict(context, folder), ensure_ascii=False)) + def _ls_json(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None: + print(json.dumps(cls._folder2dict(context, folder, files), ensure_ascii=False)) @classmethod - def _ls_plain(cls, context: LsCommandContext, folder: Folder) -> None: + def _ls_plain(cls, context: LsCommandContext, folder: Folder, files: list[File]) -> None: label = { "type": "Type", "acl": "Access", @@ -97,7 +98,7 @@ class LsCommand(BaseCommand): length["size"] = max(length["size"], len(str(folder.size))) length["date"] = max(length["date"], len(sub_folder.updated_at_name)) length["name"] = max(length["name"], len(sub_folder.name)) - for file in folder.files: + for file in files: length["size"] = max(length["size"], len(str(file.size))) length["date"] = max(length["date"], len(file.updated_at_name)) length["name"] = max(length["name"], len(file.name)) @@ -111,7 +112,7 @@ class LsCommand(BaseCommand): if context.is_recursive: print(f"{context.prefix}{folder.path}:") - print(f"total {sum(f.size for f in folder.files)}") + print(f"total {sum(f.size for f in files)}") if not context.is_quick: print(header) @@ -125,7 +126,7 @@ class LsCommand(BaseCommand): f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.size:{length['size']}}\t" f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}" ) - for file in sorted(folder.files, key=lambda x: x.name): + for file in sorted(files, key=lambda x: x.name): print( f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t" f"{context.laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t" @@ -140,12 +141,15 @@ class LsCommand(BaseCommand): if sub_folder.lock: folder_api.auth(sub_folder.id, context.password) folder = folder_api.retrieve(sub_folder.id) - cls._ls_plain(context, folder) + files = cls._find_files(context.connection, sub_folder.id) + cls._ls_plain(context, folder, files) except UnauthorizedException: pass @classmethod - def _folder2dict(cls, context: LsCommandContext, folder: Folder | FolderSimple) -> dict[str, Any]: + def _folder2dict( + cls, context: LsCommandContext, folder: Folder | FolderSimple, files: list[File] + ) -> dict[str, Any]: data: dict[str, Any] = { "id": folder.id, "pid": folder.pid, @@ -168,15 +172,16 @@ class LsCommand(BaseCommand): if sub_folder.lock: folder_api.auth(sub_folder.id, context.password) folder2 = folder_api.retrieve(sub_folder.id) - sub_folders.append(cls._folder2dict(context, folder2)) + files2 = cls._find_files(context.connection, sub_folder.id) + sub_folders.append(cls._folder2dict(context, folder2, files2)) except UnauthorizedException: pass data["sub_folders"] = sub_folders else: data["sub_folders"] = list( - map(lambda x: cls._folder2dict(context, x), sorted(folder.sub_folders, key=lambda x: x.name)) + map(lambda x: cls._folder2dict(context, x, []), sorted(folder.sub_folders, key=lambda x: x.name)) ) - data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(folder.files, key=lambda x: x.name))) + data["files"] = list(map(lambda x: cls._file2dict(context, x), sorted(files, key=lambda x: x.name))) return data @classmethod diff --git a/mdrsclient/commands/mkdir.py b/mdrsclient/commands/mkdir.py index e75d8b4..9b04f0e 100644 --- a/mdrsclient/commands/mkdir.py +++ b/mdrsclient/commands/mkdir.py @@ -6,6 +6,7 @@ from unicodedata import normalize from mdrsclient.api import FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models.file import find_file class MkdirCommand(BaseCommand): @@ -29,7 +30,8 @@ class MkdirCommand(BaseCommand): connection = cls._create_connection(remote) laboratory = cls._find_laboratory(connection, laboratory_name) parent_folder = cls._find_folder(connection, laboratory, r_dirname) - if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None: + files = cls._find_files(connection, parent_folder.id) + if parent_folder.find_sub_folder(r_basename) is not None or find_file(files, r_basename) is not None: raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.") folder_api = FoldersApi(connection) folder_api.create(normalize("NFC", r_basename), parent_folder.id) diff --git a/mdrsclient/commands/mv.py b/mdrsclient/commands/mv.py index 776f98f..2fc3884 100644 --- a/mdrsclient/commands/mv.py +++ b/mdrsclient/commands/mv.py @@ -6,6 +6,7 @@ from unicodedata import normalize from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models.file import find_file class MvCommand(BaseCommand): @@ -42,11 +43,13 @@ class MvCommand(BaseCommand): connection = cls._create_connection(s_remote) laboratory = cls._find_laboratory(connection, s_laboratory_name) s_parent_folder = cls._find_folder(connection, laboratory, s_dirname) + s_parent_files = cls._find_files(connection, s_parent_folder.id) d_parent_folder = cls._find_folder(connection, laboratory, d_dirname) - s_file = s_parent_folder.find_file(s_basename) + d_parent_files = cls._find_files(connection, d_parent_folder.id) + s_file = find_file(s_parent_files, s_basename) if s_file is not None: # source is file - d_file = d_parent_folder.find_file(d_basename) + d_file = find_file(d_parent_files, d_basename) if d_file is not None: raise IllegalArgumentException(f"File `{d_basename}` already exists.") d_sub_folder = d_parent_folder.find_sub_folder(d_basename) @@ -60,7 +63,7 @@ class MvCommand(BaseCommand): if s_folder is None: raise IllegalArgumentException(f"File or folder `{s_basename}` not found.") # source is folder - if d_parent_folder.find_file(d_basename) is not None: + if find_file(d_parent_files, d_basename) is not None: raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.") d_folder = d_parent_folder.find_sub_folder(d_basename) if d_folder is not None: diff --git a/mdrsclient/commands/rm.py b/mdrsclient/commands/rm.py index f6cd321..24dbea8 100644 --- a/mdrsclient/commands/rm.py +++ b/mdrsclient/commands/rm.py @@ -5,6 +5,7 @@ from typing import Any from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models.file import find_file class RmCommand(BaseCommand): @@ -32,7 +33,8 @@ class RmCommand(BaseCommand): connection = cls._create_connection(remote) laboratory = cls._find_laboratory(connection, laboratory_name) parent_folder = cls._find_folder(connection, laboratory, r_dirname) - file = parent_folder.find_file(r_basename) + parent_files = cls._find_files(connection, parent_folder.id) + file = find_file(parent_files, r_basename) if file is not None: file_api = FilesApi(connection) file_api.destroy(file) diff --git a/mdrsclient/commands/upload.py b/mdrsclient/commands/upload.py index 0199849..379ba65 100644 --- a/mdrsclient/commands/upload.py +++ b/mdrsclient/commands/upload.py @@ -9,13 +9,15 @@ from mdrsclient.api import FilesApi, FoldersApi from mdrsclient.commands.base import BaseCommand from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import IllegalArgumentException, MDRSException -from mdrsclient.models import Folder +from mdrsclient.models import File, Folder +from mdrsclient.models.file import find_file from mdrsclient.settings import CONCURRENT @dataclass(frozen=True) class UploadFileInfo: folder: Folder + files: list[File] path: str @@ -53,6 +55,7 @@ class UploadCommand(BaseCommand): connection = cls._create_connection(remote) laboratory = cls._find_laboratory(connection, laboratory_name) folder = cls._find_folder(connection, laboratory, r_path) + files = cls._find_files(connection, folder.id) infos: list[UploadFileInfo] = [] if os.path.isdir(l_path): if not is_recursive: @@ -60,6 +63,8 @@ class UploadCommand(BaseCommand): folder_api = FoldersApi(connection) folder_map: dict[str, Folder] = {} folder_map[r_path] = folder + files_map: dict[str, list[File]] = {} + files_map[r_path] = files l_basename = os.path.basename(l_path) for dirpath, _, filenames in os.walk(l_path, followlinks=True): sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path)) @@ -68,7 +73,10 @@ class UploadCommand(BaseCommand): # prepare destination parent path d_parent_dirname = os.path.dirname(d_dirname) if folder_map.get(d_parent_dirname) is None: - folder_map[d_parent_dirname] = cls._find_folder(connection, laboratory, d_parent_dirname) + parent_folder = cls._find_folder(connection, laboratory, d_parent_dirname) + folder_map[d_parent_dirname] = parent_folder + parent_files = cls._find_files(connection, parent_folder.id) + files_map[d_parent_dirname] = parent_files # prepare destination path if folder_map.get(d_dirname) is None: d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename) @@ -78,13 +86,16 @@ class UploadCommand(BaseCommand): d_folder_id = d_folder.id print(d_dirname) folder_map[d_dirname] = folder_api.retrieve(d_folder_id) + files_map[d_dirname] = cls._find_files(connection, d_folder_id) if d_folder is None: folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname]) # register upload file list for filename in filenames: - infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename))) + infos.append( + UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename)) + ) else: - infos.append(UploadFileInfo(folder, l_path)) + infos.append(UploadFileInfo(folder, files, l_path)) cls.__multiple_upload(connection, infos, is_skip_if_exists) @classmethod @@ -98,7 +109,7 @@ class UploadCommand(BaseCommand): @classmethod def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None: basename = os.path.basename(info.path) - file = info.folder.find_file(basename) + file = find_file(info.files, basename) try: if file is None: file_api.create(info.folder.id, info.path) diff --git a/mdrsclient/models/file.py b/mdrsclient/models/file.py index b015d11..50482e1 100644 --- a/mdrsclient/models/file.py +++ b/mdrsclient/models/file.py @@ -1,4 +1,5 @@ from typing import Any +from unicodedata import normalize from pydantic.dataclasses import dataclass @@ -25,3 +26,8 @@ class File: @property def updated_at_name(self) -> str: return iso8601_to_user_friendly(self.updated_at) + + +def find_file(files: list[File], name: str) -> File | None: + _name = normalize("NFC", name).lower() + return next((x for x in files if x.name.lower() == _name), None) diff --git a/mdrsclient/models/folder.py b/mdrsclient/models/folder.py index e5476d3..1f65a67 100644 --- a/mdrsclient/models/folder.py +++ b/mdrsclient/models/folder.py @@ -3,7 +3,6 @@ from unicodedata import normalize from pydantic.dataclasses import dataclass -from mdrsclient.models.file import File from mdrsclient.models.utils import iso8601_to_user_friendly @@ -78,13 +77,8 @@ class FolderSimple: class Folder(FolderSimple): metadata: list[dict[str, Any]] sub_folders: list[FolderSimple] - files: list[File] path: str def find_sub_folder(self, name: str) -> FolderSimple | None: _name = normalize("NFC", name).lower() return next((x for x in self.sub_folders if x.name.lower() == _name), None) - - def find_file(self, name: str) -> File | None: - _name = normalize("NFC", name).lower() - return next((x for x in self.files if x.name.lower() == _name), None) diff --git a/mdrsclient/utils.py b/mdrsclient/utils.py index b4ea263..9b088b1 100644 --- a/mdrsclient/utils.py +++ b/mdrsclient/utils.py @@ -1,5 +1,6 @@ import os from typing import IO, Any +from urllib.parse import parse_qs, urlparse if os.name == "nt": import msvcrt @@ -21,3 +22,10 @@ class FileLock: msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1) elif os.name == "posix": fcntl.flock(file.fileno(), fcntl.LOCK_UN) + + +def page_num_from_url(url: str) -> int | None: + parsed_url = urlparse(url) + params = parse_qs(parsed_url.query) + page = params.get("page", [None])[0] + return int(page) if page is not None else None