first commit

This commit is contained in:
2023-05-01 20:00:32 +09:00
commit 819c4a6a07
39 changed files with 1866 additions and 0 deletions

View File

@ -0,0 +1,13 @@
from mdrsclient.commands.config import ConfigCommand
from mdrsclient.commands.file import FileCommand
from mdrsclient.commands.folder import FolderCommand
from mdrsclient.commands.laboratory import LaboratoryCommand
from mdrsclient.commands.user import UserCommand
__all__ = [
"ConfigCommand",
"FileCommand",
"FolderCommand",
"LaboratoryCommand",
"UserCommand",
]

View File

@ -0,0 +1,11 @@
from abc import ABC, abstractmethod
from argparse import _SubParsersAction
from mdrsclient.exceptions import UnexpectedException
class BaseCommand(ABC):
@staticmethod
@abstractmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
raise UnexpectedException("Not implemented.")

View File

@ -0,0 +1,69 @@
from argparse import Namespace, _SubParsersAction
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import parse_remote_host
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import IllegalArgumentException
class ConfigCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# config
parser = top_level_subparsers.add_parser("config", help="configure remote hosts")
parser.set_defaults(func=lambda x: parser.print_help())
subparsers = parser.add_subparsers(title="config subcommands")
# config create
create_parser = subparsers.add_parser("create", help="create a new remote host")
create_parser.add_argument("remote", help="Label of remote host")
create_parser.add_argument("url", help="API entrypoint url of remote host")
create_parser.set_defaults(func=ConfigCommand.create)
# config update
update_parser = subparsers.add_parser("update", help="update a new remote host")
update_parser.add_argument("remote", help="Label of remote host")
update_parser.add_argument("url", help="API entrypoint url of remote host")
update_parser.set_defaults(func=ConfigCommand.update)
# config list
list_parser = subparsers.add_parser("list", help="list all the remote hosts")
list_parser.add_argument("-l", "--long", help="Show the api url", action="store_true")
list_parser.set_defaults(func=ConfigCommand.list)
# config delete
delete_parser = subparsers.add_parser("delete", help="delete an existing remote host")
delete_parser.add_argument("remote", help="Label of remote host")
delete_parser.set_defaults(func=ConfigCommand.delete)
@staticmethod
def create(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is not None:
raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
else:
config.url = args.url
@staticmethod
def update(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is None:
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
else:
config.url = args.url
@staticmethod
def list(args: Namespace) -> None:
config = ConfigFile("")
for remote, url in config.list():
line = f"{remote}:"
if args.long:
line += f"\t{url}"
print(line)
@staticmethod
def delete(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is None:
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
else:
del config.url

231
mdrsclient/commands/file.py Normal file
View File

@ -0,0 +1,231 @@
import dataclasses
import os
from argparse import Namespace, _SubParsersAction
from multiprocessing import Process
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
from mdrsclient.api import FileApi, FolderApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import (
create_session,
find_folder,
find_laboratory,
parse_remote_host_with_path,
)
from mdrsclient.exceptions import (
IllegalArgumentException,
MDRSException,
UnexpectedException,
)
from mdrsclient.models import File, Folder
from mdrsclient.session import MDRSSession
from mdrsclient.settings import NUMBER_OF_PROCESS
@dataclass(frozen=True)
class UploadFile:
folder: Folder
path: str
class FileCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# upload
upload_parser = top_level_subparsers.add_parser("upload", help="upload the file or directories")
upload_parser.add_argument(
"-r", "--recursive", help="Upload directories and their contents recursive", action="store_true"
)
upload_parser.add_argument("local_path", help="Local file path (/foo/bar/data.txt)")
upload_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
upload_parser.set_defaults(func=FileCommand.upload)
# download
download_parser = top_level_subparsers.add_parser("download", help="download a file")
download_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
download_parser.add_argument("local_path", help="Local folder path (/foo/bar/)")
download_parser.set_defaults(func=FileCommand.download)
# move
move_parser = top_level_subparsers.add_parser("move", help="move a file")
move_parser.add_argument("src_path", help="Source remote file path (remote:/lab/path/file)")
move_parser.add_argument("dest_path", help="Destination remote file path (remote:/lab/path/file)")
move_parser.set_defaults(func=FileCommand.move)
# remove
remove_parser = top_level_subparsers.add_parser("remove", help="remove a file")
remove_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
remove_parser.set_defaults(func=FileCommand.remove)
# file-metadata
metadata_parser = top_level_subparsers.add_parser("file-metadata", help="get the file metadata")
metadata_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
metadata_parser.set_defaults(func=FileCommand.metadata)
@staticmethod
def upload(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
local_path = os.path.realpath(args.local_path)
if not os.path.exists(local_path):
raise IllegalArgumentException(f"File or directory `{args.local_path}` not found.")
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
upload_files: list[UploadFile] = []
if os.path.isdir(local_path):
if not args.recursive:
raise IllegalArgumentException(f"Cannot upload `{args.local_path}`: Is a directory.")
folder_api = FolderApi(session)
folders: dict[str, Folder] = {}
folders[path] = folder
local_basename = os.path.basename(local_path)
for dirpath, dirnames, filenames in os.walk(local_path):
sub = (
local_basename
if dirpath == local_path
else os.path.join(local_basename, os.path.relpath(dirpath, local_path))
)
dest_folder_path = os.path.join(path, sub)
dest_folder_name = os.path.basename(dest_folder_path)
# prepare destination parent path
dest_parent_folder_path = os.path.dirname(dest_folder_path)
if folders.get(dest_parent_folder_path) is None:
res = folder_api.list(laboratory.id, dest_parent_folder_path)
if len(res) != 1:
raise UnexpectedException(f"Remote folder `{dest_parent_folder_path}` not found.")
folders[dest_parent_folder_path] = folder_api.retrieve(res[0].id)
# prepare destination path
if folders.get(dest_folder_path) is None:
dest_folder_simple = folders[dest_parent_folder_path].find_sub_folder(dest_folder_name)
if dest_folder_simple is None:
dest_folder_id = folder_api.create(dest_folder_name, folders[dest_parent_folder_path].id)
else:
dest_folder_id = dest_folder_simple.id
folders[dest_folder_path] = folder_api.retrieve(dest_folder_id)
if dest_folder_simple is None:
folders[dest_parent_folder_path].sub_folders.append(folders[dest_folder_path])
# register upload file list
for filename in filenames:
upload_files.append(UploadFile(folders[dest_folder_path], os.path.join(dirpath, filename)))
else:
upload_files.append(UploadFile(folder, local_path))
FileCommand._multiple_upload(session, upload_files)
@staticmethod
def download(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
if not os.path.isdir(args.local_path):
raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.")
local_file = os.path.join(args.local_path, file_name)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
r = session.get(session.build_url("v2/" + file.download_url), stream=True)
try:
with open(local_file, "wb") as f:
for chunk in r.iter_content(chunk_size=4096):
if chunk:
f.write(chunk)
f.flush()
except PermissionError:
raise IllegalArgumentException(f"Cannot create file `{local_file}`: Permission denied.")
@staticmethod
def move(args: Namespace) -> None:
(src_remote, src_laboratory_name, src_path) = parse_remote_host_with_path(args.src_path)
(dest_remote, dest_laboratory_name, dest_path) = parse_remote_host_with_path(args.dest_path)
if src_remote != dest_remote:
raise IllegalArgumentException("Remote host mismatched.")
if src_laboratory_name != dest_laboratory_name:
raise IllegalArgumentException("Laboratory mismatched.")
src_path = src_path.rstrip("/")
src_dirpath = os.path.dirname(src_path)
src_filename = os.path.basename(src_path)
if dest_path.endswith("/"):
dest_dirpath = dest_path
dest_filename = src_filename
else:
dest_dirpath = os.path.dirname(dest_path)
dest_filename = os.path.basename(dest_path)
session = create_session(src_remote)
laboratory = find_laboratory(session, src_laboratory_name)
src_folder = find_folder(session, laboratory, src_dirpath)
dest_folder = find_folder(session, laboratory, dest_dirpath)
src_file = src_folder.find_file(src_filename)
if src_file is None:
raise IllegalArgumentException(f"File `{src_filename}` not found.")
dest_file = dest_folder.find_file(dest_filename)
if dest_file is not None:
raise IllegalArgumentException(f"File `{dest_filename}` already exists.")
file_api = FileApi(session)
if src_folder.id != dest_folder.id:
file_api.move(src_file, dest_folder.id)
if dest_filename != src_filename:
dest_file_dict = dataclasses.asdict(src_file) | {"name": dest_filename}
dest_file = parse_obj_as(File, dest_file_dict)
file_api.update(dest_file, None)
@staticmethod
def remove(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
file_api = FileApi(session)
file_api.destroy(file)
@staticmethod
def metadata(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
file_api = FileApi(session)
metadata = file_api.metadata(file)
print(metadata)
@staticmethod
def _multiple_upload(session: MDRSSession, upload_files: list[UploadFile]) -> None:
processes: list[Process] = []
for idx in range(NUMBER_OF_PROCESS):
processes.append(
Process(
target=FileCommand._multiple_upload_worker,
args=(session, upload_files, idx, NUMBER_OF_PROCESS),
)
)
for process in processes:
process.start()
for process in processes:
process.join()
@staticmethod
def _multiple_upload_worker(session: MDRSSession, upload_files: list[UploadFile], idx: int, num_proc: int) -> None:
file_api = FileApi(session)
for upload_file in upload_files[idx::num_proc]:
file_name = os.path.basename(upload_file.path)
file = next((x for x in upload_file.folder.files if x.name == file_name), None)
try:
if file is None:
file_api.create(upload_file.folder.id, upload_file.path)
else:
file_api.update(file, upload_file.path)
pass
except MDRSException as e:
print(f"API Error: {e}")

View File

@ -0,0 +1,117 @@
import os
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import FolderApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import (
create_session,
find_folder,
find_laboratory,
parse_remote_host_with_path,
)
class FolderCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# ls
ls_parser = top_level_subparsers.add_parser("ls", help="list the folder contents")
ls_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
ls_parser.set_defaults(func=FolderCommand.list)
# mkdir
mkdir_parser = top_level_subparsers.add_parser("mkdir", help="create a new folder")
mkdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
mkdir_parser.set_defaults(func=FolderCommand.mkdir)
# rmdir
rmdir_parser = top_level_subparsers.add_parser("rmdir", help="remove a existing folder")
rmdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
rmdir_parser.set_defaults(func=FolderCommand.rmdir)
# metadata
metadata_parser = top_level_subparsers.add_parser("metadata", help="get a folder metadata")
metadata_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
metadata_parser.set_defaults(func=FolderCommand.metadata)
@staticmethod
def list(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
label = {
"type": "Type",
"acl": "Access",
"laboratory": "Laboratory",
"size": "Lock/Size",
"date": "Date",
"name": "Name",
}
length: dict[str, int] = {}
for key in label.keys():
length[key] = len(label[key])
for sub_folder in folder.sub_folders:
sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id)
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
length["size"] = max(length["size"], len(sub_folder.lock_name))
length["date"] = max(length["date"], len(sub_folder.updated_at_name))
length["name"] = max(length["name"], len(sub_folder.name))
for file in folder.files:
length["size"] = max(length["size"], len(str(file.size)))
length["date"] = max(length["date"], len(file.updated_at_name))
length["name"] = max(length["name"], len(file.name))
length["acl"] = max(length["acl"], len(folder.access_level_name))
length["laboratory"] = max(length["laboratory"], len(laboratory.name))
header = (
f"{label['type']:{length['type']}}\t{label['acl']:{length['acl']}}\t"
f"{label['laboratory']:{length['laboratory']}}\t{label['size']:{length['size']}}\t"
f"{label['date']:{length['date']}}\t{label['name']:{length['name']}}"
)
print(header)
print("-" * len(header.expandtabs()))
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id)
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
print(
f"{'[d]':{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.lock_name:{length['size']}}\t"
f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
)
for file in sorted(folder.files, key=lambda x: x.name):
print(
f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t"
f"{laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t"
f"{file.updated_at_name:{length['date']}}\t{file.name:{length['name']}}"
)
@staticmethod
def mkdir(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
folder_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
folder_api = FolderApi(session)
folder_api.create(folder_name, folder.id)
@staticmethod
def rmdir(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
folder_api = FolderApi(session)
folder_api.destroy(folder.id)
@staticmethod
def metadata(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
folder_api = FolderApi(session)
metadata = folder_api.metadata(folder.id)
print(metadata)

View File

@ -0,0 +1,42 @@
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import LaboratoryApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import create_session, parse_remote_host
class LaboratoryCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# labs
lls_parser = top_level_subparsers.add_parser("labs", help="list all laboratories")
lls_parser.add_argument("remote", help="Label of remote host")
lls_parser.set_defaults(func=LaboratoryCommand.list)
@staticmethod
def list(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
session = create_session(remote)
laboratory_api = LaboratoryApi(session)
laboratories = laboratory_api.list()
session.laboratories = laboratories
label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
length: dict[str, int] = {}
for key in label.keys():
length[key] = len(label[key])
for laboratory in laboratories:
length["id"] = max(length["id"], len(str(laboratory.id)))
length["name"] = max(length["name"], len(laboratory.name))
length["pi_name"] = max(length["pi_name"], len(laboratory.pi_name))
length["full_name"] = max(length["full_name"], len(laboratory.full_name))
header = (
f"{label['id']:{length['id']}}\t{label['name']:{length['name']}}\t"
f"{label['pi_name']:{length['pi_name']}}\t{label['full_name']:{length['full_name']}}"
)
print(header)
print("-" * len(header.expandtabs()))
for laboratory in laboratories:
print(
f"{laboratory.id:{length['id']}}\t{laboratory.name:{length['name']}}\t"
f"{laboratory.pi_name:{length['pi_name']}}\t{laboratory.full_name:{length['full_name']}}"
)

View File

@ -0,0 +1,61 @@
import getpass
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import UserApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import parse_remote_host
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import MissingConfigurationException
from mdrsclient.session import MDRSSession
class UserCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# login
login_parser = top_level_subparsers.add_parser("login", help="login to remote host")
login_parser.add_argument("remote", help="Label of remote host")
login_parser.set_defaults(func=UserCommand.login)
# logout
logout_parser = top_level_subparsers.add_parser("logout", help="logout from remote host")
logout_parser.add_argument("remote", help="Label of remote host")
logout_parser.set_defaults(func=UserCommand.logout)
# whoami
whoami_parser = top_level_subparsers.add_parser("whoami", help="show current user name")
whoami_parser.add_argument("remote", help="Label of remote host")
whoami_parser.set_defaults(func=UserCommand.whoami)
@staticmethod
def login(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
username = input("Username: ").strip()
password = getpass.getpass("Password: ").strip()
user_api = UserApi(session)
(user, token) = user_api.auth(username, password)
session.user = user
session.token = token
@staticmethod
def logout(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
session.logout()
@staticmethod
def whoami(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
if session.token is not None and session.token.is_expired:
session.logout()
username = session.user.username if session.user is not None else "(Anonymous)"
print(username)

View File

@ -0,0 +1,69 @@
import re
from mdrsclient.api import FolderApi, LaboratoryApi
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import (
IllegalArgumentException,
MissingConfigurationException,
UnauthorizedException,
UnexpectedException,
)
from mdrsclient.models import Folder, Laboratory
from mdrsclient.session import MDRSSession
def create_session(remote: str) -> MDRSSession:
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
return MDRSSession(config.remote, config.url)
def find_laboratory(session: MDRSSession, laboratory_name: str) -> Laboratory:
if session.laboratories.empty():
laboratory_api = LaboratoryApi(session)
session.laboratories = laboratory_api.list()
laboratory = session.laboratories.find_by_name(laboratory_name)
if laboratory is None:
raise IllegalArgumentException(f"Laboratory `{laboratory_name}` not found.")
return laboratory
def find_folder(session: MDRSSession, laboratory: Laboratory, path: str) -> Folder:
folder_api = FolderApi(session)
folders = folder_api.list(laboratory.id, path)
if len(folders) != 1:
raise UnexpectedException(f"Folder `{path}` not found.")
if folders[0].lock:
raise UnauthorizedException(f"Folder `{path}` is locked.")
return folder_api.retrieve(folders[0].id)
def parse_remote_host(path: str) -> str:
path_array = path.split(":")
remote_host = path_array[0]
if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2:
raise IllegalArgumentException("Invalid remote host")
return remote_host
def parse_remote_host_with_path(path: str) -> tuple[str, str, str]:
path = re.sub(r"//+|/\./+|/\.$", "/", path)
if re.search(r"/\.\./|/\.\.$", path) is not None:
raise IllegalArgumentException("Path traversal found.")
path_array = path.split(":")
if len(path_array) != 2:
raise IllegalArgumentException("Invalid remote host.")
remote_host = path_array[0]
folder_array = path_array[1].split("/")
is_absolute_path = folder_array[0] == ""
if not is_absolute_path:
raise IllegalArgumentException("Must be absolute paths.")
del folder_array[0]
if len(folder_array) == 0:
laboratory = ""
folder = ""
else:
laboratory = folder_array.pop(0)
folder = "/" + "/".join(folder_array)
return (remote_host, laboratory, folder)