From 97007233c67c23d20befc106a31a89cfe8104d39 Mon Sep 17 00:00:00 2001 From: Yoshihiro OKUMURA Date: Wed, 10 May 2023 18:17:35 +0900 Subject: [PATCH] split source code for each command. --- README.md | 14 +- mdrsclient/__main__.py | 36 ++- mdrsclient/__version__.py | 2 +- mdrsclient/commands/__init__.py | 32 +- mdrsclient/commands/base.py | 68 +++- mdrsclient/commands/config.py | 54 ++-- mdrsclient/commands/download.py | 80 +++++ mdrsclient/commands/file.py | 301 ------------------ mdrsclient/commands/file_metadata.py | 30 ++ .../commands/{laboratory.py => labs.py} | 22 +- mdrsclient/commands/login.py | 31 ++ mdrsclient/commands/logout.py | 23 ++ mdrsclient/commands/{folder.py => ls.py} | 63 +--- mdrsclient/commands/metadata.py | 22 ++ mdrsclient/commands/mkdir.py | 29 ++ mdrsclient/commands/mv.py | 76 +++++ mdrsclient/commands/rm.py | 39 +++ mdrsclient/commands/upload.py | 90 ++++++ mdrsclient/commands/user.py | 61 ---- mdrsclient/commands/utils.py | 69 ---- mdrsclient/commands/whoami.py | 29 ++ mdrsclient/settings.py | 2 +- 22 files changed, 620 insertions(+), 553 deletions(-) create mode 100644 mdrsclient/commands/download.py delete mode 100644 mdrsclient/commands/file.py create mode 100644 mdrsclient/commands/file_metadata.py rename mdrsclient/commands/{laboratory.py => labs.py} (70%) create mode 100644 mdrsclient/commands/login.py create mode 100644 mdrsclient/commands/logout.py rename mdrsclient/commands/{folder.py => ls.py} (54%) create mode 100644 mdrsclient/commands/metadata.py create mode 100644 mdrsclient/commands/mkdir.py create mode 100644 mdrsclient/commands/mv.py create mode 100644 mdrsclient/commands/rm.py create mode 100644 mdrsclient/commands/upload.py delete mode 100644 mdrsclient/commands/user.py delete mode 100644 mdrsclient/commands/utils.py create mode 100644 mdrsclient/commands/whoami.py diff --git a/README.md b/README.md index 3320102..65b7736 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ $ mdrs logout neurodata: ``` ### whoami -Print effective user name +Print current user name ``` $ mdrs whoami neurodata: ``` @@ -52,12 +52,6 @@ Create a new folder $ mdrs mkdir neurodata:/NIU/Repository/TEST ``` -### metadata -Get a folder metadata -``` -$ mdrs metadata neurodata:/NIU/Repository/TEST -``` - ### upload Upload the file or directory ``` @@ -86,6 +80,12 @@ $ mdrs rm neurodata:/NIU/Repository/TEST2/sample2.dat $ mdrs rm -r neurodata:/NIU/Repository/TEST2/dataset ``` +### metadata +Get a folder metadata +``` +$ mdrs metadata neurodata:/NIU/Repository/TEST +``` + ### file-metadata Get the file metadata ``` diff --git a/mdrsclient/__main__.py b/mdrsclient/__main__.py index 752d94b..1aeda85 100644 --- a/mdrsclient/__main__.py +++ b/mdrsclient/__main__.py @@ -2,10 +2,18 @@ import argparse from mdrsclient.commands import ( ConfigCommand, - FileCommand, - FolderCommand, - LaboratoryCommand, - UserCommand, + DownloadCommand, + FileMetadataCommand, + LabsCommand, + LoginCommand, + LogoutCommand, + LsCommand, + MetadataCommand, + MkdirCommand, + MvCommand, + RmCommand, + UploadCommand, + WhoamiCommand, ) from mdrsclient.exceptions import MDRSException @@ -14,13 +22,21 @@ def main() -> None: description = """This is a command-line program to up files.""" parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawDescriptionHelpFormatter) - subparsers = parser.add_subparsers(title="subcommands") + parsers = parser.add_subparsers(title="subcommands") - ConfigCommand.register(subparsers) - UserCommand.register(subparsers) - LaboratoryCommand.register(subparsers) - FolderCommand.register(subparsers) - FileCommand.register(subparsers) + ConfigCommand.register(parsers) + LoginCommand.register(parsers) + LogoutCommand.register(parsers) + WhoamiCommand.register(parsers) + LabsCommand.register(parsers) + LsCommand.register(parsers) + MkdirCommand.register(parsers) + UploadCommand.register(parsers) + DownloadCommand.register(parsers) + MvCommand.register(parsers) + RmCommand.register(parsers) + MetadataCommand.register(parsers) + FileMetadataCommand.register(parsers) try: args = parser.parse_args() diff --git a/mdrsclient/__version__.py b/mdrsclient/__version__.py index daf6a2d..925a68c 100644 --- a/mdrsclient/__version__.py +++ b/mdrsclient/__version__.py @@ -1,6 +1,6 @@ import os -here = os.path.abspath(os.path.dirname(__file__)) +here = os.path.realpath(os.path.dirname(__file__)) __all__ = ["__version__"] diff --git a/mdrsclient/commands/__init__.py b/mdrsclient/commands/__init__.py index 7013a53..4d907bf 100644 --- a/mdrsclient/commands/__init__.py +++ b/mdrsclient/commands/__init__.py @@ -1,13 +1,29 @@ from mdrsclient.commands.config import ConfigCommand -from mdrsclient.commands.file import FileCommand -from mdrsclient.commands.folder import FolderCommand -from mdrsclient.commands.laboratory import LaboratoryCommand -from mdrsclient.commands.user import UserCommand +from mdrsclient.commands.download import DownloadCommand +from mdrsclient.commands.file_metadata import FileMetadataCommand +from mdrsclient.commands.labs import LabsCommand +from mdrsclient.commands.login import LoginCommand +from mdrsclient.commands.logout import LogoutCommand +from mdrsclient.commands.ls import LsCommand +from mdrsclient.commands.metadata import MetadataCommand +from mdrsclient.commands.mkdir import MkdirCommand +from mdrsclient.commands.mv import MvCommand +from mdrsclient.commands.rm import RmCommand +from mdrsclient.commands.upload import UploadCommand +from mdrsclient.commands.whoami import WhoamiCommand __all__ = [ "ConfigCommand", - "FileCommand", - "FolderCommand", - "LaboratoryCommand", - "UserCommand", + "DownloadCommand", + "FileMetadataCommand", + "LabsCommand", + "LoginCommand", + "LogoutCommand", + "LsCommand", + "MetadataCommand", + "MkdirCommand", + "MvCommand", + "RmCommand", + "UploadCommand", + "WhoamiCommand", ] diff --git a/mdrsclient/commands/base.py b/mdrsclient/commands/base.py index 1442a62..fdfcec1 100644 --- a/mdrsclient/commands/base.py +++ b/mdrsclient/commands/base.py @@ -1,11 +1,73 @@ +import re from abc import ABC, abstractmethod from argparse import _SubParsersAction -from mdrsclient.exceptions import UnexpectedException +from mdrsclient.api import FolderApi, LaboratoryApi +from mdrsclient.config import ConfigFile +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import ( + IllegalArgumentException, + MissingConfigurationException, + UnauthorizedException, + UnexpectedException, +) +from mdrsclient.models import Folder, Laboratory class BaseCommand(ABC): - @staticmethod + @classmethod @abstractmethod - def register(top_level_subparsers: _SubParsersAction) -> None: + def register(cls, parsers: _SubParsersAction) -> None: raise UnexpectedException("Not implemented.") + + def _create_connection(self, remote: str) -> MDRSConnection: + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + return MDRSConnection(config.remote, config.url) + + def _find_laboratory(self, connection: MDRSConnection, name: str) -> Laboratory: + if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired: + laboratory_api = LaboratoryApi(connection) + connection.laboratories = laboratory_api.list() + laboratory = connection.laboratories.find_by_name(name) + if laboratory is None: + raise IllegalArgumentException(f"Laboratory `{name}` not found.") + return laboratory + + def _find_folder(self, connection: MDRSConnection, laboratory: Laboratory, path: str) -> Folder: + folder_api = FolderApi(connection) + folders = folder_api.list(laboratory.id, path) + if len(folders) != 1: + raise UnexpectedException(f"Folder `{path}` not found.") + if folders[0].lock: + raise UnauthorizedException(f"Folder `{path}` is locked.") + return folder_api.retrieve(folders[0].id) + + def _parse_remote_host(self, path: str) -> str: + path_array = path.split(":") + remote_host = path_array[0] + if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2: + raise IllegalArgumentException("Invalid remote host") + return remote_host + + def _parse_remote_host_with_path(self, path: str) -> tuple[str, str, str]: + path = re.sub(r"//+|/\./+|/\.$", "/", path) + if re.search(r"/\.\./|/\.\.$", path) is not None: + raise IllegalArgumentException("Path traversal found.") + path_array = path.split(":") + if len(path_array) != 2: + raise IllegalArgumentException("Invalid remote host.") + remote_host = path_array[0] + folder_array = path_array[1].split("/") + is_absolute_path = folder_array[0] == "" + if not is_absolute_path: + raise IllegalArgumentException("Must be absolute paths.") + del folder_array[0] + if len(folder_array) == 0: + laboratory = "" + folder = "" + else: + laboratory = folder_array.pop(0) + folder = "/" + "/".join(folder_array) + return (remote_host, laboratory, folder) diff --git a/mdrsclient/commands/config.py b/mdrsclient/commands/config.py index 84442a2..50f148e 100644 --- a/mdrsclient/commands/config.py +++ b/mdrsclient/commands/config.py @@ -1,57 +1,54 @@ from argparse import Namespace, _SubParsersAction from mdrsclient.commands.base import BaseCommand -from mdrsclient.commands.utils import parse_remote_host from mdrsclient.config import ConfigFile from mdrsclient.exceptions import IllegalArgumentException class ConfigCommand(BaseCommand): - @staticmethod - def register(top_level_subparsers: _SubParsersAction) -> None: + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() # config - parser = top_level_subparsers.add_parser("config", help="configure remote hosts") - parser.set_defaults(func=lambda x: parser.print_help()) - subparsers = parser.add_subparsers(title="config subcommands") + config_parser = parsers.add_parser("config", help="configure remote hosts") + config_parser.set_defaults(func=lambda x: config_parser.print_help()) + config_parsers = config_parser.add_subparsers(title="config subcommands") # config create - create_parser = subparsers.add_parser("create", help="create a new remote host") - create_parser.add_argument("remote", help="Label of remote host") + create_parser = config_parsers.add_parser("create", help="create a new remote host") + create_parser.add_argument("remote", help="label of remote host") create_parser.add_argument("url", help="API entrypoint url of remote host") - create_parser.set_defaults(func=ConfigCommand.create) + create_parser.set_defaults(func=command.create) # config update - update_parser = subparsers.add_parser("update", help="update a new remote host") - update_parser.add_argument("remote", help="Label of remote host") + update_parser = config_parsers.add_parser("update", help="update a new remote host") + update_parser.add_argument("remote", help="label of remote host") update_parser.add_argument("url", help="API entrypoint url of remote host") - update_parser.set_defaults(func=ConfigCommand.update) + update_parser.set_defaults(func=command.update) # config list - list_parser = subparsers.add_parser("list", help="list all the remote hosts") - list_parser.add_argument("-l", "--long", help="Show the api url", action="store_true") - list_parser.set_defaults(func=ConfigCommand.list) + list_parser = config_parsers.add_parser("list", help="list all the remote hosts") + list_parser.add_argument("-l", "--long", help="show the api url", action="store_true") + list_parser.set_defaults(func=command.list) # config delete - delete_parser = subparsers.add_parser("delete", help="delete an existing remote host") - delete_parser.add_argument("remote", help="Label of remote host") - delete_parser.set_defaults(func=ConfigCommand.delete) + delete_parser = config_parsers.add_parser("delete", help="delete an existing remote host") + delete_parser.add_argument("remote", help="label of remote host") + delete_parser.set_defaults(func=command.delete) - @staticmethod - def create(args: Namespace) -> None: - remote = parse_remote_host(args.remote) + def create(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) config = ConfigFile(remote=remote) if config.url is not None: raise IllegalArgumentException(f"Remote host `{remote}` is already exists.") else: config.url = args.url - @staticmethod - def update(args: Namespace) -> None: - remote = parse_remote_host(args.remote) + def update(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) config = ConfigFile(remote=remote) if config.url is None: raise IllegalArgumentException(f"Remote host `{remote}` is not exists.") else: config.url = args.url - @staticmethod - def list(args: Namespace) -> None: + def list(self, args: Namespace) -> None: config = ConfigFile("") for remote, url in config.list(): line = f"{remote}:" @@ -59,9 +56,8 @@ class ConfigCommand(BaseCommand): line += f"\t{url}" print(line) - @staticmethod - def delete(args: Namespace) -> None: - remote = parse_remote_host(args.remote) + def delete(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) config = ConfigFile(remote=remote) if config.url is None: raise IllegalArgumentException(f"Remote host `{remote}` is not exists.") diff --git a/mdrsclient/commands/download.py b/mdrsclient/commands/download.py new file mode 100644 index 0000000..7c36eb9 --- /dev/null +++ b/mdrsclient/commands/download.py @@ -0,0 +1,80 @@ +import os +from argparse import Namespace, _SubParsersAction +from concurrent.futures import ThreadPoolExecutor + +from pydantic.dataclasses import dataclass + +from mdrsclient.api import FileApi, FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models import File +from mdrsclient.settings import CONCURRENT + + +@dataclass(frozen=True) +class DownloadFileInfo: + file: File + path: str + + +class DownloadCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + download_parser = parsers.add_parser("download", help="download the file or folder") + download_parser.add_argument( + "-r", "--recursive", help="download folders and their contents recursive", action="store_true" + ) + download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)") + download_parser.add_argument("local_path", help="local folder path (/foo/bar/)") + download_parser.set_defaults(func=command.download) + + def download(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + r_path = r_path.rstrip("/") + r_dirname = os.path.dirname(r_path) + r_basename = os.path.basename(r_path) + connection = self._create_connection(remote) + l_dirname = os.path.realpath(args.local_path) + if not os.path.isdir(l_dirname): + raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.") + laboratory = self._find_laboratory(connection, laboratory_name) + r_parent_folder = self._find_folder(connection, laboratory, r_dirname) + file = r_parent_folder.find_file(r_basename) + download_files: list[DownloadFileInfo] = [] + if file is not None: + l_path = os.path.join(l_dirname, r_basename) + download_files.append(DownloadFileInfo(file, l_path)) + else: + folder = r_parent_folder.find_sub_folder(r_basename) + if folder is None: + raise IllegalArgumentException(f"File or folder `{r_path}` not found.") + if not args.recursive: + raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.") + folder_api = FolderApi(connection) + self.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname) + self.__multiple_download(connection, download_files) + + def __multiple_download_pickup_recursive_files( + self, folder_api: FolderApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str + ) -> None: + folder = folder_api.retrieve(folder_id) + dirname = os.path.join(basedir, folder.name) + if not os.path.exists(dirname): + os.makedirs(dirname) + print(dirname) + for file in folder.files: + path = os.path.join(dirname, file.name) + infolist.append(DownloadFileInfo(file, path)) + for sub_folder in folder.sub_folders: + self.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname) + + def __multiple_download(self, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None: + file_api = FileApi(connection) + with ThreadPoolExecutor(max_workers=CONCURRENT) as pool: + pool.map(lambda x: self.__multiple_download_worker(file_api, x), infolist) + + def __multiple_download_worker(self, file_api: FileApi, info: DownloadFileInfo) -> None: + file_api.download(info.file, info.path) + print(info.path) diff --git a/mdrsclient/commands/file.py b/mdrsclient/commands/file.py deleted file mode 100644 index ab345b3..0000000 --- a/mdrsclient/commands/file.py +++ /dev/null @@ -1,301 +0,0 @@ -import dataclasses -import os -from argparse import Namespace, _SubParsersAction -from concurrent.futures import ThreadPoolExecutor - -from pydantic import parse_obj_as -from pydantic.dataclasses import dataclass - -from mdrsclient.api import FileApi, FolderApi -from mdrsclient.commands.base import BaseCommand -from mdrsclient.commands.utils import ( - create_connection, - find_folder, - find_laboratory, - parse_remote_host_with_path, -) -from mdrsclient.connection import MDRSConnection -from mdrsclient.exceptions import ( - IllegalArgumentException, - MDRSException, - UnexpectedException, -) -from mdrsclient.models import File, Folder, FolderSimple -from mdrsclient.settings import NUMBER_OF_PROCESS - - -@dataclass(frozen=True) -class UploadFile: - folder: Folder - path: str - - -@dataclass(frozen=True) -class DownloadFile: - file: File - path: str - - -class FileCommand(BaseCommand): - @staticmethod - def register(top_level_subparsers: _SubParsersAction) -> None: - # upload - upload_parser = top_level_subparsers.add_parser("upload", help="upload the file or directory") - upload_parser.add_argument( - "-r", "--recursive", help="Upload directories and their contents recursive", action="store_true" - ) - upload_parser.add_argument("local_path", help="Local file path (/foo/bar/data.txt)") - upload_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") - upload_parser.set_defaults(func=FileCommand.upload) - # download - download_parser = top_level_subparsers.add_parser("download", help="download the file or folder") - download_parser.add_argument( - "-r", "--recursive", help="Download folders and their contents recursive", action="store_true" - ) - download_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") - download_parser.add_argument("local_path", help="Local folder path (/foo/bar/)") - download_parser.set_defaults(func=FileCommand.download) - # mv - move_parser = top_level_subparsers.add_parser("mv", help="move or rename the file or folder") - move_parser.add_argument("src_path", help="Source remote path (remote:/lab/path/src)") - move_parser.add_argument("dest_path", help="Destination remote path (remote:/lab/path/dest)") - move_parser.set_defaults(func=FileCommand.move) - # rm - remove_parser = top_level_subparsers.add_parser("rm", help="remove the file or folder") - remove_parser.add_argument( - "-r", "--recursive", help="Remove folders and their contents recursive", action="store_true" - ) - remove_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") - remove_parser.set_defaults(func=FileCommand.remove) - # file-metadata - metadata_parser = top_level_subparsers.add_parser("file-metadata", help="get the file metadata") - metadata_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") - metadata_parser.set_defaults(func=FileCommand.metadata) - - @staticmethod - def upload(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - local_path = os.path.realpath(args.local_path) - if not os.path.exists(local_path): - raise IllegalArgumentException(f"File or directory `{args.local_path}` not found.") - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - folder = find_folder(connection, laboratory, path) - upload_files: list[UploadFile] = [] - if os.path.isdir(local_path): - if not args.recursive: - raise IllegalArgumentException(f"Cannot upload `{args.local_path}`: Is a directory.") - folder_api = FolderApi(connection) - folders: dict[str, Folder] = {} - folders[path] = folder - local_basename = os.path.basename(local_path) - for dirpath, dirnames, filenames in os.walk(local_path): - sub = ( - local_basename - if dirpath == local_path - else os.path.join(local_basename, os.path.relpath(dirpath, local_path)) - ) - dest_folder_path = os.path.join(path, sub) - dest_folder_name = os.path.basename(dest_folder_path) - # prepare destination parent path - dest_parent_folder_path = os.path.dirname(dest_folder_path) - if folders.get(dest_parent_folder_path) is None: - res = folder_api.list(laboratory.id, dest_parent_folder_path) - if len(res) != 1: - raise UnexpectedException(f"Remote folder `{dest_parent_folder_path}` not found.") - folders[dest_parent_folder_path] = folder_api.retrieve(res[0].id) - # prepare destination path - if folders.get(dest_folder_path) is None: - dest_folder_simple = folders[dest_parent_folder_path].find_sub_folder(dest_folder_name) - if dest_folder_simple is None: - dest_folder_id = folder_api.create(dest_folder_name, folders[dest_parent_folder_path].id) - else: - dest_folder_id = dest_folder_simple.id - print(dest_folder_path) - folders[dest_folder_path] = folder_api.retrieve(dest_folder_id) - if dest_folder_simple is None: - folders[dest_parent_folder_path].sub_folders.append(folders[dest_folder_path]) - # register upload file list - for filename in filenames: - upload_files.append(UploadFile(folders[dest_folder_path], os.path.join(dirpath, filename))) - else: - upload_files.append(UploadFile(folder, local_path)) - FileCommand._multiple_upload(connection, upload_files) - - @staticmethod - def download(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - path = path.rstrip("/") - parent_path = os.path.dirname(path) - file_name = os.path.basename(path) - connection = create_connection(remote) - local_path = os.path.abspath(args.local_path) - if not os.path.isdir(local_path): - raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.") - laboratory = find_laboratory(connection, laboratory_name) - parent_folder = find_folder(connection, laboratory, parent_path) - file = parent_folder.find_file(file_name) - download_files: list[DownloadFile] = [] - if file is not None: - file_path = os.path.join(local_path, file_name) - download_files.append(DownloadFile(file, file_path)) - else: - sub_folder = parent_folder.find_sub_folder(file_name) - if sub_folder is None: - raise IllegalArgumentException(f"File or Folder`{file_name}` not found.") - if not args.recursive: - raise IllegalArgumentException(f"Cannot download `{path}`: Is a folder.") - folder_api = FolderApi(connection) - sub_folder_dirname = os.path.join(local_path, sub_folder.name) - FileCommand._multiple_download_pickup_recursive_files( - folder_api, download_files, sub_folder.id, sub_folder_dirname - ) - FileCommand._multiple_download(connection, download_files) - - @staticmethod - def move(args: Namespace) -> None: - (src_remote, src_laboratory_name, src_path) = parse_remote_host_with_path(args.src_path) - (dest_remote, dest_laboratory_name, dest_path) = parse_remote_host_with_path(args.dest_path) - if src_remote != dest_remote: - raise IllegalArgumentException("Remote host mismatched.") - if src_laboratory_name != dest_laboratory_name: - raise IllegalArgumentException("Laboratory mismatched.") - src_path = src_path.rstrip("/") - src_dirname = os.path.dirname(src_path) - src_basename = os.path.basename(src_path) - if dest_path.endswith("/"): - dest_dirname = dest_path - dest_basename = src_basename - else: - dest_dirname = os.path.dirname(dest_path) - dest_basename = os.path.basename(dest_path) - connection = create_connection(src_remote) - laboratory = find_laboratory(connection, src_laboratory_name) - src_parent_folder = find_folder(connection, laboratory, src_dirname) - dest_parent_folder = find_folder(connection, laboratory, dest_dirname) - src_file = src_parent_folder.find_file(src_basename) - if src_file is not None: - # source is file - dest_file = dest_parent_folder.find_file(dest_basename) - if dest_file is not None: - raise IllegalArgumentException(f"File `{dest_basename}` already exists.") - dest_sub_folder = dest_parent_folder.find_sub_folder(dest_basename) - if dest_sub_folder is not None: - raise IllegalArgumentException( - f"Cannot overwrite non-folder `{dest_basename}` with folder `{dest_path}`." - ) - file_api = FileApi(connection) - if src_parent_folder.id != dest_parent_folder.id: - file_api.move(src_file, dest_parent_folder.id) - if dest_basename != src_basename: - dest_file_dict = dataclasses.asdict(src_file) | {"name": dest_basename} - dest_file = parse_obj_as(File, dest_file_dict) - file_api.update(dest_file, None) - else: - src_folder = src_parent_folder.find_sub_folder(src_basename) - if src_folder is None: - raise IllegalArgumentException(f"File or Folder `{src_basename}` not found.") - # source is folder - dest_file = dest_parent_folder.find_file(dest_basename) - if dest_file is not None: - raise IllegalArgumentException( - f"Cannot overwrite non-folder `{dest_basename}` with folder `{src_path}`." - ) - dest_folder = dest_parent_folder.find_sub_folder(dest_basename) - if dest_folder is not None: - if dest_folder.id == src_folder.id: - raise IllegalArgumentException(f"`{src_path}` and `{src_path}` are the same folder.") - raise IllegalArgumentException(f"Cannot move `{src_path}` to `{dest_path}`: Folder not empty.") - folder_api = FolderApi(connection) - if src_parent_folder.id != dest_parent_folder.id: - folder_api.move(src_folder, dest_parent_folder.id) - if src_basename != dest_basename: - dest_folder_dict = dataclasses.asdict(src_folder) | {"name": dest_basename} - dest_folder = parse_obj_as(FolderSimple, dest_folder_dict) - folder_api.update(dest_folder) - - @staticmethod - def remove(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - path = path.rstrip("/") - parent_path = os.path.dirname(path) - file_name = os.path.basename(path) - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - parent_folder = find_folder(connection, laboratory, parent_path) - file = parent_folder.find_file(file_name) - if file is not None: - file_api = FileApi(connection) - file_api.destroy(file) - else: - folder = parent_folder.find_sub_folder(file_name) - if folder is None: - raise IllegalArgumentException(f"File `{file_name}` not found.") - if not args.recursive: - raise IllegalArgumentException(f"Cannot remove `{path}`: Is a folder.") - folder_api = FolderApi(connection) - folder_api.destroy(folder.id) - - @staticmethod - def metadata(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - path = path.rstrip("/") - parent_path = os.path.dirname(path) - file_name = os.path.basename(path) - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - folder = find_folder(connection, laboratory, parent_path) - file = folder.find_file(file_name) - if file is None: - raise IllegalArgumentException(f"File `{file_name}` not found.") - file_api = FileApi(connection) - metadata = file_api.metadata(file) - print(metadata) - - @staticmethod - def _multiple_upload(connection: MDRSConnection, upload_files: list[UploadFile]) -> None: - file_api = FileApi(connection) - with ThreadPoolExecutor(max_workers=NUMBER_OF_PROCESS) as pool: - pool.map(lambda x: FileCommand._multiple_upload_worker(file_api, x), upload_files) - - @staticmethod - def _multiple_upload_worker(file_api: FileApi, upload_file: UploadFile) -> None: - file_name = os.path.basename(upload_file.path) - file = next((x for x in upload_file.folder.files if x.name == file_name), None) - try: - if file is None: - file_api.create(upload_file.folder.id, upload_file.path) - else: - file_api.update(file, upload_file.path) - print(upload_file.path) - except MDRSException as e: - print(f"API Error: {e}") - - @staticmethod - def _multiple_download_pickup_recursive_files( - folder_api: FolderApi, download_files: list[DownloadFile], folder_id: str, local_dirname: str - ) -> None: - folder = folder_api.retrieve(folder_id) - file_dirname = os.path.join(local_dirname, folder.name) - if not os.path.exists(file_dirname): - os.makedirs(file_dirname) - print(file_dirname) - for file in folder.files: - file_path = os.path.join(file_dirname, file.name) - download_files.append(DownloadFile(file, file_path)) - for sub_folder in folder.sub_folders: - sub_folder_dirname = os.path.join(local_dirname, sub_folder.name) - FileCommand._multiple_download_pickup_recursive_files( - folder_api, download_files, sub_folder.id, sub_folder_dirname - ) - - @staticmethod - def _multiple_download(connection: MDRSConnection, download_files: list[DownloadFile]) -> None: - file_api = FileApi(connection) - with ThreadPoolExecutor(max_workers=NUMBER_OF_PROCESS) as pool: - pool.map(lambda x: FileCommand._multiple_download_worker(file_api, x), download_files) - - @staticmethod - def _multiple_download_worker(file_api: FileApi, download_file: DownloadFile) -> None: - file_api.download(download_file.file, download_file.path) - print(download_file.path) diff --git a/mdrsclient/commands/file_metadata.py b/mdrsclient/commands/file_metadata.py new file mode 100644 index 0000000..5ebc670 --- /dev/null +++ b/mdrsclient/commands/file_metadata.py @@ -0,0 +1,30 @@ +import os +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import FileApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.exceptions import IllegalArgumentException + + +class FileMetadataCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + file_metadata_parser = parsers.add_parser("file-metadata", help="get the file metadata") + file_metadata_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)") + file_metadata_parser.set_defaults(func=command.file_metadata) + + def file_metadata(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + r_path = r_path.rstrip("/") + r_dirname = os.path.dirname(r_path) + r_basename = os.path.basename(r_path) + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + folder = self._find_folder(connection, laboratory, r_dirname) + file = folder.find_file(r_basename) + if file is None: + raise IllegalArgumentException(f"File `{r_basename}` not found.") + file_api = FileApi(connection) + metadata = file_api.metadata(file) + print(metadata) diff --git a/mdrsclient/commands/laboratory.py b/mdrsclient/commands/labs.py similarity index 70% rename from mdrsclient/commands/laboratory.py rename to mdrsclient/commands/labs.py index f3e1663..f2b65e7 100644 --- a/mdrsclient/commands/laboratory.py +++ b/mdrsclient/commands/labs.py @@ -2,21 +2,19 @@ from argparse import Namespace, _SubParsersAction from mdrsclient.api import LaboratoryApi from mdrsclient.commands.base import BaseCommand -from mdrsclient.commands.utils import create_connection, parse_remote_host -class LaboratoryCommand(BaseCommand): - @staticmethod - def register(top_level_subparsers: _SubParsersAction) -> None: - # labs - lls_parser = top_level_subparsers.add_parser("labs", help="list all laboratories") - lls_parser.add_argument("remote", help="Label of remote host") - lls_parser.set_defaults(func=LaboratoryCommand.list) +class LabsCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + labs_parser = parsers.add_parser("labs", help="list all laboratories") + labs_parser.add_argument("remote", help="label of remote host") + labs_parser.set_defaults(func=command.labs) - @staticmethod - def list(args: Namespace) -> None: - remote = parse_remote_host(args.remote) - connection = create_connection(remote) + def labs(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) + connection = self._create_connection(remote) laboratory_api = LaboratoryApi(connection) laboratories = laboratory_api.list() connection.laboratories = laboratories diff --git a/mdrsclient/commands/login.py b/mdrsclient/commands/login.py new file mode 100644 index 0000000..3e5a832 --- /dev/null +++ b/mdrsclient/commands/login.py @@ -0,0 +1,31 @@ +import getpass +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import UserApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.config import ConfigFile +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import MissingConfigurationException + + +class LoginCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + login_parser = parsers.add_parser("login", help="login to remote host") + login_parser.add_argument("remote", help="label of remote host") + login_parser.set_defaults(func=command.login) + + def login(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + connection = MDRSConnection(config.remote, config.url) + username = input("Username: ").strip() + password = getpass.getpass("Password: ").strip() + user_api = UserApi(connection) + (user, token) = user_api.auth(username, password) + print("Login Successful") + connection.user = user + connection.token = token diff --git a/mdrsclient/commands/logout.py b/mdrsclient/commands/logout.py new file mode 100644 index 0000000..4bc9a7a --- /dev/null +++ b/mdrsclient/commands/logout.py @@ -0,0 +1,23 @@ +from argparse import Namespace, _SubParsersAction + +from mdrsclient.commands.base import BaseCommand +from mdrsclient.config import ConfigFile +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import MissingConfigurationException + + +class LogoutCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + logout_parser = parsers.add_parser("logout", help="logout from remote host") + logout_parser.add_argument("remote", help="label of remote host") + logout_parser.set_defaults(func=command.logout) + + def logout(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + connection = MDRSConnection(config.remote, config.url) + connection.logout() diff --git a/mdrsclient/commands/folder.py b/mdrsclient/commands/ls.py similarity index 54% rename from mdrsclient/commands/folder.py rename to mdrsclient/commands/ls.py index ab5ede7..0f58e62 100644 --- a/mdrsclient/commands/folder.py +++ b/mdrsclient/commands/ls.py @@ -1,38 +1,21 @@ -import os from argparse import Namespace, _SubParsersAction -from mdrsclient.api import FolderApi from mdrsclient.commands.base import BaseCommand -from mdrsclient.commands.utils import ( - create_connection, - find_folder, - find_laboratory, - parse_remote_host_with_path, -) -class FolderCommand(BaseCommand): - @staticmethod - def register(top_level_subparsers: _SubParsersAction) -> None: - # ls - ls_parser = top_level_subparsers.add_parser("ls", help="list the folder contents") - ls_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") - ls_parser.set_defaults(func=FolderCommand.list) - # mkdir - mkdir_parser = top_level_subparsers.add_parser("mkdir", help="create a new folder") - mkdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") - mkdir_parser.set_defaults(func=FolderCommand.mkdir) - # metadata - metadata_parser = top_level_subparsers.add_parser("metadata", help="get a folder metadata") - metadata_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") - metadata_parser.set_defaults(func=FolderCommand.metadata) +class LsCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + ls_parser = parsers.add_parser("ls", help="list the folder contents") + ls_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)") + ls_parser.set_defaults(func=command.ls) - @staticmethod - def list(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - folder = find_folder(connection, laboratory, path) + def ls(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + folder = self._find_folder(connection, laboratory, r_path) label = { "type": "Type", "acl": "Access", @@ -80,25 +63,3 @@ class FolderCommand(BaseCommand): f"{laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t" f"{file.updated_at_name:{length['date']}}\t{file.name:{length['name']}}" ) - - @staticmethod - def mkdir(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - path = path.rstrip("/") - parent_path = os.path.dirname(path) - folder_name = os.path.basename(path) - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - folder = find_folder(connection, laboratory, parent_path) - folder_api = FolderApi(connection) - folder_api.create(folder_name, folder.id) - - @staticmethod - def metadata(args: Namespace) -> None: - (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) - connection = create_connection(remote) - laboratory = find_laboratory(connection, laboratory_name) - folder = find_folder(connection, laboratory, path) - folder_api = FolderApi(connection) - metadata = folder_api.metadata(folder.id) - print(metadata) diff --git a/mdrsclient/commands/metadata.py b/mdrsclient/commands/metadata.py new file mode 100644 index 0000000..c73d7b2 --- /dev/null +++ b/mdrsclient/commands/metadata.py @@ -0,0 +1,22 @@ +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import FolderApi +from mdrsclient.commands.base import BaseCommand + + +class MetadataCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + metadata_parser = parsers.add_parser("metadata", help="get a folder metadata") + metadata_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)") + metadata_parser.set_defaults(func=command.metadata) + + def metadata(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + folder = self._find_folder(connection, laboratory, r_path) + folder_api = FolderApi(connection) + metadata = folder_api.metadata(folder.id) + print(metadata) diff --git a/mdrsclient/commands/mkdir.py b/mdrsclient/commands/mkdir.py new file mode 100644 index 0000000..18467f7 --- /dev/null +++ b/mdrsclient/commands/mkdir.py @@ -0,0 +1,29 @@ +import os +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.exceptions import IllegalArgumentException + + +class MkdirCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + # mkdir + mkdir_parser = parsers.add_parser("mkdir", help="create a new folder") + mkdir_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)") + mkdir_parser.set_defaults(func=command.mkdir) + + def mkdir(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + r_path = r_path.rstrip("/") + r_dirname = os.path.dirname(r_path) + r_basename = os.path.basename(r_path) + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + parent_folder = self._find_folder(connection, laboratory, r_dirname) + if parent_folder.find_sub_folder(r_basename) is not None or parent_folder.find_file(r_basename) is not None: + raise IllegalArgumentException(f"Cannot create folder `{r_path}`: File exists.") + folder_api = FolderApi(connection) + folder_api.create(r_basename, parent_folder.id) diff --git a/mdrsclient/commands/mv.py b/mdrsclient/commands/mv.py new file mode 100644 index 0000000..36fdaba --- /dev/null +++ b/mdrsclient/commands/mv.py @@ -0,0 +1,76 @@ +import dataclasses +import os +from argparse import Namespace, _SubParsersAction + +from pydantic import parse_obj_as + +from mdrsclient.api import FileApi, FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.models import File, FolderSimple + + +class MvCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + mv_parser = parsers.add_parser("mv", help="move or rename the file or folder") + mv_parser.add_argument("src_path", help="source remote path (remote:/lab/path/src)") + mv_parser.add_argument("dest_path", help="destination remote path (remote:/lab/path/dest)") + mv_parser.set_defaults(func=command.mv) + + def mv(self, args: Namespace) -> None: + (s_remote, s_laboratory_name, s_path) = self._parse_remote_host_with_path(args.src_path) + (d_remote, d_laboratory_name, d_path) = self._parse_remote_host_with_path(args.dest_path) + if s_remote != d_remote: + raise IllegalArgumentException("Remote host mismatched.") + if s_laboratory_name != d_laboratory_name: + raise IllegalArgumentException("Laboratory mismatched.") + s_path = s_path.rstrip("/") + s_dirname = os.path.dirname(s_path) + s_basename = os.path.basename(s_path) + if d_path.endswith("/"): + d_dirname = d_path + d_basename = s_basename + else: + d_dirname = os.path.dirname(d_path) + d_basename = os.path.basename(d_path) + connection = self._create_connection(s_remote) + laboratory = self._find_laboratory(connection, s_laboratory_name) + s_parent_folder = self._find_folder(connection, laboratory, s_dirname) + d_parent_folder = self._find_folder(connection, laboratory, d_dirname) + s_file = s_parent_folder.find_file(s_basename) + if s_file is not None: + # source is file + d_file = d_parent_folder.find_file(d_basename) + if d_file is not None: + raise IllegalArgumentException(f"File `{d_basename}` already exists.") + d_sub_folder = d_parent_folder.find_sub_folder(d_basename) + if d_sub_folder is not None: + raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{d_path}`.") + file_api = FileApi(connection) + if s_parent_folder.id != d_parent_folder.id: + file_api.move(s_file, d_parent_folder.id) + if d_basename != s_basename: + d_file_dict = dataclasses.asdict(s_file) | {"name": d_basename} + d_file = parse_obj_as(File, d_file_dict) + file_api.update(d_file, None) + else: + s_folder = s_parent_folder.find_sub_folder(s_basename) + if s_folder is None: + raise IllegalArgumentException(f"File or folder `{s_basename}` not found.") + # source is folder + if d_parent_folder.find_file(d_basename) is not None: + raise IllegalArgumentException(f"Cannot overwrite non-folder `{d_basename}` with folder `{s_path}`.") + d_folder = d_parent_folder.find_sub_folder(d_basename) + if d_folder is not None: + if d_folder.id == s_folder.id: + raise IllegalArgumentException(f"`{s_path}` and `{s_path}` are the same folder.") + raise IllegalArgumentException(f"Cannot move `{s_path}` to `{d_path}`: Folder not empty.") + folder_api = FolderApi(connection) + if s_parent_folder.id != d_parent_folder.id: + folder_api.move(s_folder, d_parent_folder.id) + if s_basename != d_basename: + d_folder_dict = dataclasses.asdict(s_folder) | {"name": d_basename} + d_folder = parse_obj_as(FolderSimple, d_folder_dict) + folder_api.update(d_folder) diff --git a/mdrsclient/commands/rm.py b/mdrsclient/commands/rm.py new file mode 100644 index 0000000..621081f --- /dev/null +++ b/mdrsclient/commands/rm.py @@ -0,0 +1,39 @@ +import os +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import FileApi, FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.exceptions import IllegalArgumentException + + +class RmCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + rm_parser = parsers.add_parser("rm", help="remove the file or folder") + rm_parser.add_argument( + "-r", "--recursive", help="remove folders and their contents recursive", action="store_true" + ) + rm_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)") + rm_parser.set_defaults(func=command.rm) + + def rm(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + r_path = r_path.rstrip("/") + r_dirname = os.path.dirname(r_path) + r_basename = os.path.basename(r_path) + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + parent_folder = self._find_folder(connection, laboratory, r_dirname) + file = parent_folder.find_file(r_basename) + if file is not None: + file_api = FileApi(connection) + file_api.destroy(file) + else: + folder = parent_folder.find_sub_folder(r_basename) + if folder is None: + raise IllegalArgumentException(f"Cannot remove `{r_path}`: No such file or folder.") + if not args.recursive: + raise IllegalArgumentException(f"Cannot remove `{r_path}`: Is a folder.") + folder_api = FolderApi(connection) + folder_api.destroy(folder.id) diff --git a/mdrsclient/commands/upload.py b/mdrsclient/commands/upload.py new file mode 100644 index 0000000..b5bb3b5 --- /dev/null +++ b/mdrsclient/commands/upload.py @@ -0,0 +1,90 @@ +import os +from argparse import Namespace, _SubParsersAction +from concurrent.futures import ThreadPoolExecutor + +from pydantic.dataclasses import dataclass + +from mdrsclient.api import FileApi, FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import IllegalArgumentException, MDRSException +from mdrsclient.models import Folder +from mdrsclient.settings import CONCURRENT + + +@dataclass(frozen=True) +class UploadFileInfo: + folder: Folder + path: str + + +class UploadCommand(BaseCommand): + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + upload_parser = parsers.add_parser("upload", help="upload the file or directory") + upload_parser.add_argument( + "-r", "--recursive", help="upload directories and their contents recursive", action="store_true" + ) + upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)") + upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)") + upload_parser.set_defaults(func=command.upload) + + def upload(self, args: Namespace) -> None: + (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) + l_path = os.path.realpath(args.local_path) + if not os.path.exists(l_path): + raise IllegalArgumentException(f"File or directory `{args.local_path}` not found.") + connection = self._create_connection(remote) + laboratory = self._find_laboratory(connection, laboratory_name) + folder = self._find_folder(connection, laboratory, r_path) + infos: list[UploadFileInfo] = [] + if os.path.isdir(l_path): + if not args.recursive: + raise IllegalArgumentException(f"Cannot upload `{args.local_path}`: Is a directory.") + folder_api = FolderApi(connection) + folder_map: dict[str, Folder] = {} + folder_map[r_path] = folder + l_basename = os.path.basename(l_path) + for dirpath, dirnames, filenames in os.walk(l_path): + sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path)) + d_dirname = os.path.join(r_path, sub) + d_basename = os.path.basename(d_dirname) + # prepare destination parent path + d_parent_dirname = os.path.dirname(d_dirname) + if folder_map.get(d_parent_dirname) is None: + folder_map[d_parent_dirname] = self._find_folder(connection, laboratory, d_parent_dirname) + # prepare destination path + if folder_map.get(d_dirname) is None: + d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename) + if d_folder is None: + d_folder_id = folder_api.create(d_basename, folder_map[d_parent_dirname].id) + else: + d_folder_id = d_folder.id + print(d_dirname) + folder_map[d_dirname] = folder_api.retrieve(d_folder_id) + if d_folder is None: + folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname]) + # register upload file list + for filename in filenames: + infos.append(UploadFileInfo(folder_map[d_dirname], os.path.join(dirpath, filename))) + else: + infos.append(UploadFileInfo(folder, l_path)) + self.__multiple_upload(connection, infos) + + def __multiple_upload(self, connection: MDRSConnection, infos: list[UploadFileInfo]) -> None: + file_api = FileApi(connection) + with ThreadPoolExecutor(max_workers=CONCURRENT) as pool: + pool.map(lambda x: self.__multiple_upload_worker(file_api, x), infos) + + def __multiple_upload_worker(self, file_api: FileApi, info: UploadFileInfo) -> None: + basename = os.path.basename(info.path) + file = info.folder.find_file(basename) + try: + if file is None: + file_api.create(info.folder.id, info.path) + else: + file_api.update(file, info.path) + print(os.path.join(info.folder.path, basename)) + except MDRSException as e: + print(f"API Error: {e}") diff --git a/mdrsclient/commands/user.py b/mdrsclient/commands/user.py deleted file mode 100644 index 540d4d7..0000000 --- a/mdrsclient/commands/user.py +++ /dev/null @@ -1,61 +0,0 @@ -import getpass -from argparse import Namespace, _SubParsersAction - -from mdrsclient.api import UserApi -from mdrsclient.commands.base import BaseCommand -from mdrsclient.commands.utils import parse_remote_host -from mdrsclient.config import ConfigFile -from mdrsclient.connection import MDRSConnection -from mdrsclient.exceptions import MissingConfigurationException - - -class UserCommand(BaseCommand): - @staticmethod - def register(top_level_subparsers: _SubParsersAction) -> None: - # login - login_parser = top_level_subparsers.add_parser("login", help="login to remote host") - login_parser.add_argument("remote", help="Label of remote host") - login_parser.set_defaults(func=UserCommand.login) - # logout - logout_parser = top_level_subparsers.add_parser("logout", help="logout from remote host") - logout_parser.add_argument("remote", help="Label of remote host") - logout_parser.set_defaults(func=UserCommand.logout) - # whoami - whoami_parser = top_level_subparsers.add_parser("whoami", help="show current user name") - whoami_parser.add_argument("remote", help="Label of remote host") - whoami_parser.set_defaults(func=UserCommand.whoami) - - @staticmethod - def login(args: Namespace) -> None: - remote = parse_remote_host(args.remote) - config = ConfigFile(remote) - if config.url is None: - raise MissingConfigurationException(f"Remote host `{remote}` is not found.") - connection = MDRSConnection(config.remote, config.url) - username = input("Username: ").strip() - password = getpass.getpass("Password: ").strip() - user_api = UserApi(connection) - (user, token) = user_api.auth(username, password) - connection.user = user - connection.token = token - - @staticmethod - def logout(args: Namespace) -> None: - remote = parse_remote_host(args.remote) - config = ConfigFile(remote) - if config.url is None: - raise MissingConfigurationException(f"Remote host `{remote}` is not found.") - connection = MDRSConnection(config.remote, config.url) - connection.logout() - - @staticmethod - def whoami(args: Namespace) -> None: - remote = parse_remote_host(args.remote) - config = ConfigFile(remote) - if config.url is None: - raise MissingConfigurationException(f"Remote host `{remote}` is not found.") - connection = MDRSConnection(config.remote, config.url) - if connection.token is not None and connection.token.is_expired: - connection.logout() - username = connection.user.username if connection.user is not None else "(Anonymous)" - print(username) diff --git a/mdrsclient/commands/utils.py b/mdrsclient/commands/utils.py deleted file mode 100644 index 7761efd..0000000 --- a/mdrsclient/commands/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -import re - -from mdrsclient.api import FolderApi, LaboratoryApi -from mdrsclient.config import ConfigFile -from mdrsclient.connection import MDRSConnection -from mdrsclient.exceptions import ( - IllegalArgumentException, - MissingConfigurationException, - UnauthorizedException, - UnexpectedException, -) -from mdrsclient.models import Folder, Laboratory - - -def create_connection(remote: str) -> MDRSConnection: - config = ConfigFile(remote) - if config.url is None: - raise MissingConfigurationException(f"Remote host `{remote}` is not found.") - return MDRSConnection(config.remote, config.url) - - -def find_laboratory(connection: MDRSConnection, laboratory_name: str) -> Laboratory: - if connection.laboratories.empty() or connection.token is not None and connection.token.is_expired: - laboratory_api = LaboratoryApi(connection) - connection.laboratories = laboratory_api.list() - laboratory = connection.laboratories.find_by_name(laboratory_name) - if laboratory is None: - raise IllegalArgumentException(f"Laboratory `{laboratory_name}` not found.") - return laboratory - - -def find_folder(connection: MDRSConnection, laboratory: Laboratory, path: str) -> Folder: - folder_api = FolderApi(connection) - folders = folder_api.list(laboratory.id, path) - if len(folders) != 1: - raise UnexpectedException(f"Folder `{path}` not found.") - if folders[0].lock: - raise UnauthorizedException(f"Folder `{path}` is locked.") - return folder_api.retrieve(folders[0].id) - - -def parse_remote_host(path: str) -> str: - path_array = path.split(":") - remote_host = path_array[0] - if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2: - raise IllegalArgumentException("Invalid remote host") - return remote_host - - -def parse_remote_host_with_path(path: str) -> tuple[str, str, str]: - path = re.sub(r"//+|/\./+|/\.$", "/", path) - if re.search(r"/\.\./|/\.\.$", path) is not None: - raise IllegalArgumentException("Path traversal found.") - path_array = path.split(":") - if len(path_array) != 2: - raise IllegalArgumentException("Invalid remote host.") - remote_host = path_array[0] - folder_array = path_array[1].split("/") - is_absolute_path = folder_array[0] == "" - if not is_absolute_path: - raise IllegalArgumentException("Must be absolute paths.") - del folder_array[0] - if len(folder_array) == 0: - laboratory = "" - folder = "" - else: - laboratory = folder_array.pop(0) - folder = "/" + "/".join(folder_array) - return (remote_host, laboratory, folder) diff --git a/mdrsclient/commands/whoami.py b/mdrsclient/commands/whoami.py new file mode 100644 index 0000000..5d53c75 --- /dev/null +++ b/mdrsclient/commands/whoami.py @@ -0,0 +1,29 @@ +from argparse import Namespace, _SubParsersAction +from typing import Final + +from mdrsclient.commands.base import BaseCommand +from mdrsclient.config import ConfigFile +from mdrsclient.connection import MDRSConnection +from mdrsclient.exceptions import MissingConfigurationException + + +class WhoamiCommand(BaseCommand): + ANONYMOUS_USERNAME: Final[str] = "(Anonymous)" + + @classmethod + def register(cls, parsers: _SubParsersAction) -> None: + command = cls() + whoami_parser = parsers.add_parser("whoami", help="show current user name") + whoami_parser.add_argument("remote", help="label of remote host") + whoami_parser.set_defaults(func=command.whoami) + + def whoami(self, args: Namespace) -> None: + remote = self._parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + connection = MDRSConnection(config.remote, config.url) + if connection.token is not None and connection.token.is_expired: + connection.logout() + username = connection.user.username if connection.user is not None else self.ANONYMOUS_USERNAME + print(username) diff --git a/mdrsclient/settings.py b/mdrsclient/settings.py index 5e07159..997be6b 100644 --- a/mdrsclient/settings.py +++ b/mdrsclient/settings.py @@ -15,4 +15,4 @@ class Settings(BaseSettings): settings = Settings() CONCURRENT = settings.concurrent -CONFIG_DIRNAME = os.path.abspath(os.path.expanduser(settings.config_dirname)) +CONFIG_DIRNAME = os.path.realpath(os.path.expanduser(settings.config_dirname))