first commit

This commit is contained in:
2023-05-01 20:00:32 +09:00
commit 819c4a6a07
39 changed files with 1866 additions and 0 deletions

0
mdrsclient/__init__.py Normal file
View File

37
mdrsclient/__main__.py Normal file
View File

@ -0,0 +1,37 @@
import argparse
from mdrsclient.commands import (
ConfigCommand,
FileCommand,
FolderCommand,
LaboratoryCommand,
UserCommand,
)
from mdrsclient.exceptions import MDRSException
def main() -> None:
description = """This is a command-line program to up files."""
parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawDescriptionHelpFormatter)
subparsers = parser.add_subparsers(title="subcommands")
ConfigCommand.register(subparsers)
UserCommand.register(subparsers)
LaboratoryCommand.register(subparsers)
FolderCommand.register(subparsers)
FileCommand.register(subparsers)
try:
args = parser.parse_args()
if hasattr(args, "func"):
args.func(args)
else:
parser.print_help()
except MDRSException as e:
print(f"Error: {e}")
exit(2)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,8 @@
import os
here = os.path.abspath(os.path.dirname(__file__))
__all__ = ["__version__"]
with open(os.path.join(os.path.dirname(here), "VERSION")) as version_file:
__version__ = version_file.read().strip()

View File

@ -0,0 +1,11 @@
from mdrsclient.api.file import FileApi
from mdrsclient.api.folder import FolderApi
from mdrsclient.api.laboratory import LaboratoryApi
from mdrsclient.api.user import UserApi
__all__ = [
"FileApi",
"FolderApi",
"LaboratoryApi",
"UserApi",
]

51
mdrsclient/api/base.py Normal file
View File

@ -0,0 +1,51 @@
from abc import ABC
import requests
from pydantic import parse_obj_as
from requests import Response
from mdrsclient.exceptions import (
BadRequestException,
ForbiddenException,
UnauthorizedException,
UnexpectedException,
)
from mdrsclient.models import DRFStandardizedErrors
from mdrsclient.session import MDRSSession
class BaseApi(ABC):
def __init__(self, session: MDRSSession) -> None:
self.session = session
def _get(self, url, *args, **kwargs) -> Response:
return self.session.get(self.__build_url(url), *args, **kwargs)
def _post(self, url, *args, **kwargs) -> Response:
return self.session.post(self.__build_url(url), *args, **kwargs)
def _put(self, url, *args, **kwargs) -> Response:
return self.session.put(self.__build_url(url), *args, **kwargs)
def _delete(self, url, *args, **kwargs) -> Response:
return self.session.delete(self.__build_url(url), *args, **kwargs)
def _patch(self, url, *args, **kwargs) -> Response:
return self.session.patch(self.__build_url(url), *args, **kwargs)
def _raise_response_error(self, response: Response) -> None:
if response.status_code >= 300:
if response.status_code < 400 or response.status_code >= 500:
raise UnexpectedException(f"Unexpected status code returned: {response.status_code}.")
errors = parse_obj_as(DRFStandardizedErrors, response.json())
if response.status_code == requests.codes.bad_request:
raise BadRequestException(errors.errors[0].detail)
elif response.status_code == requests.codes.unauthorized:
raise UnauthorizedException("Login required.")
elif response.status_code == requests.codes.forbidden:
raise ForbiddenException("You do not have enough permissions. Access is denied.")
else:
raise UnexpectedException(errors.errors[0].detail)
def __build_url(self, *args: tuple) -> str:
return self.session.build_url(*args)

82
mdrsclient/api/file.py Normal file
View File

@ -0,0 +1,82 @@
import sys
from typing import Final
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
from mdrsclient.api.base import BaseApi
from mdrsclient.api.utils import token_check
from mdrsclient.exceptions import UnexpectedException
from mdrsclient.models import File
@dataclass(frozen=True)
class FileCreateResponse:
id: str
class FileApi(BaseApi):
ENTRYPOINT: Final[str] = "v2/file/"
def retrieve(self, id: str) -> File:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + id + "/"
token_check(self.session)
response = self._get(url)
self._raise_response_error(response)
return parse_obj_as(File, response.json())
def create(self, folder_id: str, path: str) -> str:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT
token_check(self.session)
data = {"folder_id": folder_id}
try:
with open(path, mode="rb") as fp:
response = self._post(url, data=data, files={"file": fp})
self._raise_response_error(response)
ret = parse_obj_as(FileCreateResponse, response.json())
except OSError:
raise UnexpectedException(f"Could not open `{path}` file.")
return ret.id
def update(self, file: File, path: str | None) -> bool:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + file.id + "/"
token_check(self.session)
if path is not None:
try:
with open(path, mode="rb") as fp:
response = self._put(url, files={"file": fp})
except OSError:
raise UnexpectedException(f"Could not open `{path}` file.")
else:
data = {"name": file.name, "description": file.description}
response = self._put(url, data=data)
self._raise_response_error(response)
return True
def destroy(self, file: File) -> bool:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + file.id + "/"
token_check(self.session)
response = self._delete(url)
self._raise_response_error(response)
return True
def move(self, file: File, folder_id: str | None) -> bool:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + file.id + "/move/"
data = {"folder": folder_id}
token_check(self.session)
response = self._post(url, data=data)
self._raise_response_error(response)
return True
def metadata(self, file: File) -> dict:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + file.id + "/metadata/"
token_check(self.session)
response = self._get(url)
self._raise_response_error(response)
return response.json()

77
mdrsclient/api/folder.py Normal file
View File

@ -0,0 +1,77 @@
import sys
from typing import Final
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
from mdrsclient.api.base import BaseApi
from mdrsclient.api.utils import token_check
from mdrsclient.models import Folder, FolderSimple
@dataclass(frozen=True)
class FolderCreateResponse:
id: str
class FolderApi(BaseApi):
ENTRYPOINT: Final[str] = "v2/folder/"
def list(self, laboratory_id: int, path: str) -> list[FolderSimple]:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT
params = {"path": path, "laboratory_id": laboratory_id}
token_check(self.session)
response = self._get(url, params=params)
self._raise_response_error(response)
ret: list[FolderSimple] = []
for data in response.json():
ret.append(parse_obj_as(FolderSimple, data))
return ret
def retrieve(self, id: str) -> Folder:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + id + "/"
token_check(self.session)
response = self._get(url)
self._raise_response_error(response)
ret = parse_obj_as(Folder, response.json())
return ret
def create(self, name: str, parent_id: str) -> str:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT
data = {"name": name, "parent_id": parent_id, "description": "", "template_id": -1}
token_check(self.session)
response = self._post(url, data=data)
self._raise_response_error(response)
ret = parse_obj_as(FolderCreateResponse, response.json())
return ret.id
def update(self, folder: FolderSimple) -> bool:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT
data = {
"name": folder.name,
"description": folder.description,
}
token_check(self.session)
response = self._put(url, data=data)
self._raise_response_error(response)
return True
def destroy(self, id: str) -> bool:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + id + "/"
token_check(self.session)
response = self._delete(url)
self._raise_response_error(response)
return True
def metadata(self, id: str) -> dict:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + id + "/metadata/"
token_check(self.session)
response = self._get(url)
self._raise_response_error(response)
return response.json()

View File

@ -0,0 +1,23 @@
import sys
from typing import Final
from pydantic import parse_obj_as
from mdrsclient.api.base import BaseApi
from mdrsclient.api.utils import token_check
from mdrsclient.models import Laboratories, Laboratory
class LaboratoryApi(BaseApi):
ENTRYPOINT: Final[str] = "v2/laboratory/"
def list(self) -> Laboratories:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT
token_check(self.session)
response = self._get(url)
self._raise_response_error(response)
ret = Laboratories([])
for data in response.json():
ret.append(parse_obj_as(Laboratory, data))
return ret

44
mdrsclient/api/user.py Normal file
View File

@ -0,0 +1,44 @@
import sys
from typing import Final
import requests
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
from mdrsclient.api.base import BaseApi
from mdrsclient.exceptions import UnauthorizedException
from mdrsclient.models import Token, User
@dataclass(frozen=True)
class UserAuthResponse(Token):
laboratory: str
lab_id: int
class UserApi(BaseApi):
ENTRYPOINT: Final[str] = "v2/"
def auth(self, username: str, password: str) -> tuple[User, Token]:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + "auth/"
data = {"username": username, "password": password}
response = self._post(url, data=data)
if response.status_code == requests.codes.unauthorized:
raise UnauthorizedException("Invalid username or password.")
self._raise_response_error(response)
obj = parse_obj_as(UserAuthResponse, response.json())
token = Token(access=obj.access, refresh=obj.refresh)
user = User(id=token.user_id, username=username, laboratory_id=obj.lab_id, laboratory=obj.laboratory)
return (user, token)
def refresh(self, token: Token) -> Token:
print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
url = self.ENTRYPOINT + "refresh/"
data = {"refresh": token.refresh}
response = self._post(url, data=data)
if response.status_code == requests.codes.unauthorized:
raise UnauthorizedException("Token is invalid or expired.")
self._raise_response_error(response)
token = parse_obj_as(Token, response.json())
return token

15
mdrsclient/api/utils.py Normal file
View File

@ -0,0 +1,15 @@
from mdrsclient.api.user import UserApi
from mdrsclient.exceptions import UnauthorizedException
from mdrsclient.session import MDRSSession
def token_check(session: MDRSSession) -> None:
if session.token is not None:
if session.token.is_refresh_required:
user_api = UserApi(session)
try:
session.token = user_api.refresh(session.token)
except UnauthorizedException:
session.logout()
elif session.token.is_expired:
session.logout()

113
mdrsclient/cache.py Normal file
View File

@ -0,0 +1,113 @@
import dataclasses
import fcntl
import json
import os
from pydantic import ValidationError
from pydantic.dataclasses import dataclass
from pydantic.tools import parse_obj_as
from mdrsclient.models import Laboratories, Token, User
from mdrsclient.settings import CONFIG_DIR_PATH
@dataclass
class CacheData:
user: User | None
token: Token | None
laboratories: Laboratories
class CacheFile:
serial: int
cache_dir: str
cache_file: str
data: CacheData
def __init__(self, remote: str) -> None:
self.serial = -1
self.cache_dir = os.path.join(CONFIG_DIR_PATH, "cache")
self.cache_file = os.path.join(self.cache_dir, remote + ".json")
self.data = CacheData(user=None, token=None, laboratories=Laboratories([]))
def dump(self) -> CacheData | None:
self.__load()
return self.data
@property
def token(self) -> Token | None:
self.__load()
return self.data.token
@token.setter
def token(self, token: Token) -> None:
self.__load()
self.data.token = token
self.__save()
@token.deleter
def token(self) -> None:
if self.data.token is not None:
self.__clear()
@property
def user(self) -> User | None:
return self.data.user
@user.setter
def user(self, user: User) -> None:
self.__load()
self.data.user = user
self.__save()
@user.deleter
def user(self) -> None:
if self.data.user is not None:
self.__clear()
@property
def laboratories(self) -> Laboratories:
return self.data.laboratories
@laboratories.setter
def laboratories(self, laboratories: Laboratories) -> None:
self.__load()
self.data.laboratories = laboratories
self.__save()
def __clear(self) -> None:
self.data.user = None
self.data.token = None
self.data.laboratories.clear()
self.__save()
def __load(self) -> None:
if os.path.isfile(self.cache_file):
stat = os.stat(self.cache_file)
serial = hash((stat.st_uid, stat.st_gid, stat.st_mode, stat.st_size, stat.st_mtime))
if self.serial != serial:
try:
with open(self.cache_file) as f:
self.data = parse_obj_as(CacheData, json.load(f))
except ValidationError:
self.__clear()
self.__save()
else:
self.serial = serial
else:
self.data.token = None
self.serial = -1
def __save(self) -> None:
self.__ensure_cache_dir()
with open(self.cache_file, "w") as f:
fcntl.flock(f, fcntl.LOCK_EX)
f.write(json.dumps(dataclasses.asdict(self.data)))
# ensure file is secure.
os.chmod(self.cache_file, 0o600)
def __ensure_cache_dir(self) -> None:
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
# ensure directory is secure.
os.chmod(self.cache_dir, 0o700)

View File

@ -0,0 +1,13 @@
from mdrsclient.commands.config import ConfigCommand
from mdrsclient.commands.file import FileCommand
from mdrsclient.commands.folder import FolderCommand
from mdrsclient.commands.laboratory import LaboratoryCommand
from mdrsclient.commands.user import UserCommand
__all__ = [
"ConfigCommand",
"FileCommand",
"FolderCommand",
"LaboratoryCommand",
"UserCommand",
]

View File

@ -0,0 +1,11 @@
from abc import ABC, abstractmethod
from argparse import _SubParsersAction
from mdrsclient.exceptions import UnexpectedException
class BaseCommand(ABC):
@staticmethod
@abstractmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
raise UnexpectedException("Not implemented.")

View File

@ -0,0 +1,69 @@
from argparse import Namespace, _SubParsersAction
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import parse_remote_host
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import IllegalArgumentException
class ConfigCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# config
parser = top_level_subparsers.add_parser("config", help="configure remote hosts")
parser.set_defaults(func=lambda x: parser.print_help())
subparsers = parser.add_subparsers(title="config subcommands")
# config create
create_parser = subparsers.add_parser("create", help="create a new remote host")
create_parser.add_argument("remote", help="Label of remote host")
create_parser.add_argument("url", help="API entrypoint url of remote host")
create_parser.set_defaults(func=ConfigCommand.create)
# config update
update_parser = subparsers.add_parser("update", help="update a new remote host")
update_parser.add_argument("remote", help="Label of remote host")
update_parser.add_argument("url", help="API entrypoint url of remote host")
update_parser.set_defaults(func=ConfigCommand.update)
# config list
list_parser = subparsers.add_parser("list", help="list all the remote hosts")
list_parser.add_argument("-l", "--long", help="Show the api url", action="store_true")
list_parser.set_defaults(func=ConfigCommand.list)
# config delete
delete_parser = subparsers.add_parser("delete", help="delete an existing remote host")
delete_parser.add_argument("remote", help="Label of remote host")
delete_parser.set_defaults(func=ConfigCommand.delete)
@staticmethod
def create(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is not None:
raise IllegalArgumentException(f"Remote host `{remote}` is already exists.")
else:
config.url = args.url
@staticmethod
def update(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is None:
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
else:
config.url = args.url
@staticmethod
def list(args: Namespace) -> None:
config = ConfigFile("")
for remote, url in config.list():
line = f"{remote}:"
if args.long:
line += f"\t{url}"
print(line)
@staticmethod
def delete(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote=remote)
if config.url is None:
raise IllegalArgumentException(f"Remote host `{remote}` is not exists.")
else:
del config.url

231
mdrsclient/commands/file.py Normal file
View File

@ -0,0 +1,231 @@
import dataclasses
import os
from argparse import Namespace, _SubParsersAction
from multiprocessing import Process
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
from mdrsclient.api import FileApi, FolderApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import (
create_session,
find_folder,
find_laboratory,
parse_remote_host_with_path,
)
from mdrsclient.exceptions import (
IllegalArgumentException,
MDRSException,
UnexpectedException,
)
from mdrsclient.models import File, Folder
from mdrsclient.session import MDRSSession
from mdrsclient.settings import NUMBER_OF_PROCESS
@dataclass(frozen=True)
class UploadFile:
folder: Folder
path: str
class FileCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# upload
upload_parser = top_level_subparsers.add_parser("upload", help="upload the file or directories")
upload_parser.add_argument(
"-r", "--recursive", help="Upload directories and their contents recursive", action="store_true"
)
upload_parser.add_argument("local_path", help="Local file path (/foo/bar/data.txt)")
upload_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
upload_parser.set_defaults(func=FileCommand.upload)
# download
download_parser = top_level_subparsers.add_parser("download", help="download a file")
download_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
download_parser.add_argument("local_path", help="Local folder path (/foo/bar/)")
download_parser.set_defaults(func=FileCommand.download)
# move
move_parser = top_level_subparsers.add_parser("move", help="move a file")
move_parser.add_argument("src_path", help="Source remote file path (remote:/lab/path/file)")
move_parser.add_argument("dest_path", help="Destination remote file path (remote:/lab/path/file)")
move_parser.set_defaults(func=FileCommand.move)
# remove
remove_parser = top_level_subparsers.add_parser("remove", help="remove a file")
remove_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
remove_parser.set_defaults(func=FileCommand.remove)
# file-metadata
metadata_parser = top_level_subparsers.add_parser("file-metadata", help="get the file metadata")
metadata_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)")
metadata_parser.set_defaults(func=FileCommand.metadata)
@staticmethod
def upload(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
local_path = os.path.realpath(args.local_path)
if not os.path.exists(local_path):
raise IllegalArgumentException(f"File or directory `{args.local_path}` not found.")
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
upload_files: list[UploadFile] = []
if os.path.isdir(local_path):
if not args.recursive:
raise IllegalArgumentException(f"Cannot upload `{args.local_path}`: Is a directory.")
folder_api = FolderApi(session)
folders: dict[str, Folder] = {}
folders[path] = folder
local_basename = os.path.basename(local_path)
for dirpath, dirnames, filenames in os.walk(local_path):
sub = (
local_basename
if dirpath == local_path
else os.path.join(local_basename, os.path.relpath(dirpath, local_path))
)
dest_folder_path = os.path.join(path, sub)
dest_folder_name = os.path.basename(dest_folder_path)
# prepare destination parent path
dest_parent_folder_path = os.path.dirname(dest_folder_path)
if folders.get(dest_parent_folder_path) is None:
res = folder_api.list(laboratory.id, dest_parent_folder_path)
if len(res) != 1:
raise UnexpectedException(f"Remote folder `{dest_parent_folder_path}` not found.")
folders[dest_parent_folder_path] = folder_api.retrieve(res[0].id)
# prepare destination path
if folders.get(dest_folder_path) is None:
dest_folder_simple = folders[dest_parent_folder_path].find_sub_folder(dest_folder_name)
if dest_folder_simple is None:
dest_folder_id = folder_api.create(dest_folder_name, folders[dest_parent_folder_path].id)
else:
dest_folder_id = dest_folder_simple.id
folders[dest_folder_path] = folder_api.retrieve(dest_folder_id)
if dest_folder_simple is None:
folders[dest_parent_folder_path].sub_folders.append(folders[dest_folder_path])
# register upload file list
for filename in filenames:
upload_files.append(UploadFile(folders[dest_folder_path], os.path.join(dirpath, filename)))
else:
upload_files.append(UploadFile(folder, local_path))
FileCommand._multiple_upload(session, upload_files)
@staticmethod
def download(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
if not os.path.isdir(args.local_path):
raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.")
local_file = os.path.join(args.local_path, file_name)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
r = session.get(session.build_url("v2/" + file.download_url), stream=True)
try:
with open(local_file, "wb") as f:
for chunk in r.iter_content(chunk_size=4096):
if chunk:
f.write(chunk)
f.flush()
except PermissionError:
raise IllegalArgumentException(f"Cannot create file `{local_file}`: Permission denied.")
@staticmethod
def move(args: Namespace) -> None:
(src_remote, src_laboratory_name, src_path) = parse_remote_host_with_path(args.src_path)
(dest_remote, dest_laboratory_name, dest_path) = parse_remote_host_with_path(args.dest_path)
if src_remote != dest_remote:
raise IllegalArgumentException("Remote host mismatched.")
if src_laboratory_name != dest_laboratory_name:
raise IllegalArgumentException("Laboratory mismatched.")
src_path = src_path.rstrip("/")
src_dirpath = os.path.dirname(src_path)
src_filename = os.path.basename(src_path)
if dest_path.endswith("/"):
dest_dirpath = dest_path
dest_filename = src_filename
else:
dest_dirpath = os.path.dirname(dest_path)
dest_filename = os.path.basename(dest_path)
session = create_session(src_remote)
laboratory = find_laboratory(session, src_laboratory_name)
src_folder = find_folder(session, laboratory, src_dirpath)
dest_folder = find_folder(session, laboratory, dest_dirpath)
src_file = src_folder.find_file(src_filename)
if src_file is None:
raise IllegalArgumentException(f"File `{src_filename}` not found.")
dest_file = dest_folder.find_file(dest_filename)
if dest_file is not None:
raise IllegalArgumentException(f"File `{dest_filename}` already exists.")
file_api = FileApi(session)
if src_folder.id != dest_folder.id:
file_api.move(src_file, dest_folder.id)
if dest_filename != src_filename:
dest_file_dict = dataclasses.asdict(src_file) | {"name": dest_filename}
dest_file = parse_obj_as(File, dest_file_dict)
file_api.update(dest_file, None)
@staticmethod
def remove(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
file_api = FileApi(session)
file_api.destroy(file)
@staticmethod
def metadata(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
file_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
file = folder.find_file(file_name)
if file is None:
raise IllegalArgumentException(f"File `{file_name}` not found.")
file_api = FileApi(session)
metadata = file_api.metadata(file)
print(metadata)
@staticmethod
def _multiple_upload(session: MDRSSession, upload_files: list[UploadFile]) -> None:
processes: list[Process] = []
for idx in range(NUMBER_OF_PROCESS):
processes.append(
Process(
target=FileCommand._multiple_upload_worker,
args=(session, upload_files, idx, NUMBER_OF_PROCESS),
)
)
for process in processes:
process.start()
for process in processes:
process.join()
@staticmethod
def _multiple_upload_worker(session: MDRSSession, upload_files: list[UploadFile], idx: int, num_proc: int) -> None:
file_api = FileApi(session)
for upload_file in upload_files[idx::num_proc]:
file_name = os.path.basename(upload_file.path)
file = next((x for x in upload_file.folder.files if x.name == file_name), None)
try:
if file is None:
file_api.create(upload_file.folder.id, upload_file.path)
else:
file_api.update(file, upload_file.path)
pass
except MDRSException as e:
print(f"API Error: {e}")

View File

@ -0,0 +1,117 @@
import os
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import FolderApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import (
create_session,
find_folder,
find_laboratory,
parse_remote_host_with_path,
)
class FolderCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# ls
ls_parser = top_level_subparsers.add_parser("ls", help="list the folder contents")
ls_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
ls_parser.set_defaults(func=FolderCommand.list)
# mkdir
mkdir_parser = top_level_subparsers.add_parser("mkdir", help="create a new folder")
mkdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
mkdir_parser.set_defaults(func=FolderCommand.mkdir)
# rmdir
rmdir_parser = top_level_subparsers.add_parser("rmdir", help="remove a existing folder")
rmdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
rmdir_parser.set_defaults(func=FolderCommand.rmdir)
# metadata
metadata_parser = top_level_subparsers.add_parser("metadata", help="get a folder metadata")
metadata_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)")
metadata_parser.set_defaults(func=FolderCommand.metadata)
@staticmethod
def list(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
label = {
"type": "Type",
"acl": "Access",
"laboratory": "Laboratory",
"size": "Lock/Size",
"date": "Date",
"name": "Name",
}
length: dict[str, int] = {}
for key in label.keys():
length[key] = len(label[key])
for sub_folder in folder.sub_folders:
sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id)
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
length["acl"] = max(length["acl"], len(sub_folder.access_level_name))
length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name))
length["size"] = max(length["size"], len(sub_folder.lock_name))
length["date"] = max(length["date"], len(sub_folder.updated_at_name))
length["name"] = max(length["name"], len(sub_folder.name))
for file in folder.files:
length["size"] = max(length["size"], len(str(file.size)))
length["date"] = max(length["date"], len(file.updated_at_name))
length["name"] = max(length["name"], len(file.name))
length["acl"] = max(length["acl"], len(folder.access_level_name))
length["laboratory"] = max(length["laboratory"], len(laboratory.name))
header = (
f"{label['type']:{length['type']}}\t{label['acl']:{length['acl']}}\t"
f"{label['laboratory']:{length['laboratory']}}\t{label['size']:{length['size']}}\t"
f"{label['date']:{length['date']}}\t{label['name']:{length['name']}}"
)
print(header)
print("-" * len(header.expandtabs()))
for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name):
sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id)
sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)"
print(
f"{'[d]':{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t"
f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.lock_name:{length['size']}}\t"
f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}"
)
for file in sorted(folder.files, key=lambda x: x.name):
print(
f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t"
f"{laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t"
f"{file.updated_at_name:{length['date']}}\t{file.name:{length['name']}}"
)
@staticmethod
def mkdir(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
path = path.rstrip("/")
parent_path = os.path.dirname(path)
folder_name = os.path.basename(path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, parent_path)
folder_api = FolderApi(session)
folder_api.create(folder_name, folder.id)
@staticmethod
def rmdir(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
folder_api = FolderApi(session)
folder_api.destroy(folder.id)
@staticmethod
def metadata(args: Namespace) -> None:
(remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path)
session = create_session(remote)
laboratory = find_laboratory(session, laboratory_name)
folder = find_folder(session, laboratory, path)
folder_api = FolderApi(session)
metadata = folder_api.metadata(folder.id)
print(metadata)

View File

@ -0,0 +1,42 @@
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import LaboratoryApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import create_session, parse_remote_host
class LaboratoryCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# labs
lls_parser = top_level_subparsers.add_parser("labs", help="list all laboratories")
lls_parser.add_argument("remote", help="Label of remote host")
lls_parser.set_defaults(func=LaboratoryCommand.list)
@staticmethod
def list(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
session = create_session(remote)
laboratory_api = LaboratoryApi(session)
laboratories = laboratory_api.list()
session.laboratories = laboratories
label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"}
length: dict[str, int] = {}
for key in label.keys():
length[key] = len(label[key])
for laboratory in laboratories:
length["id"] = max(length["id"], len(str(laboratory.id)))
length["name"] = max(length["name"], len(laboratory.name))
length["pi_name"] = max(length["pi_name"], len(laboratory.pi_name))
length["full_name"] = max(length["full_name"], len(laboratory.full_name))
header = (
f"{label['id']:{length['id']}}\t{label['name']:{length['name']}}\t"
f"{label['pi_name']:{length['pi_name']}}\t{label['full_name']:{length['full_name']}}"
)
print(header)
print("-" * len(header.expandtabs()))
for laboratory in laboratories:
print(
f"{laboratory.id:{length['id']}}\t{laboratory.name:{length['name']}}\t"
f"{laboratory.pi_name:{length['pi_name']}}\t{laboratory.full_name:{length['full_name']}}"
)

View File

@ -0,0 +1,61 @@
import getpass
from argparse import Namespace, _SubParsersAction
from mdrsclient.api import UserApi
from mdrsclient.commands.base import BaseCommand
from mdrsclient.commands.utils import parse_remote_host
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import MissingConfigurationException
from mdrsclient.session import MDRSSession
class UserCommand(BaseCommand):
@staticmethod
def register(top_level_subparsers: _SubParsersAction) -> None:
# login
login_parser = top_level_subparsers.add_parser("login", help="login to remote host")
login_parser.add_argument("remote", help="Label of remote host")
login_parser.set_defaults(func=UserCommand.login)
# logout
logout_parser = top_level_subparsers.add_parser("logout", help="logout from remote host")
logout_parser.add_argument("remote", help="Label of remote host")
logout_parser.set_defaults(func=UserCommand.logout)
# whoami
whoami_parser = top_level_subparsers.add_parser("whoami", help="show current user name")
whoami_parser.add_argument("remote", help="Label of remote host")
whoami_parser.set_defaults(func=UserCommand.whoami)
@staticmethod
def login(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
username = input("Username: ").strip()
password = getpass.getpass("Password: ").strip()
user_api = UserApi(session)
(user, token) = user_api.auth(username, password)
session.user = user
session.token = token
@staticmethod
def logout(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
session.logout()
@staticmethod
def whoami(args: Namespace) -> None:
remote = parse_remote_host(args.remote)
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
session = MDRSSession(config.remote, config.url)
if session.token is not None and session.token.is_expired:
session.logout()
username = session.user.username if session.user is not None else "(Anonymous)"
print(username)

View File

@ -0,0 +1,69 @@
import re
from mdrsclient.api import FolderApi, LaboratoryApi
from mdrsclient.config import ConfigFile
from mdrsclient.exceptions import (
IllegalArgumentException,
MissingConfigurationException,
UnauthorizedException,
UnexpectedException,
)
from mdrsclient.models import Folder, Laboratory
from mdrsclient.session import MDRSSession
def create_session(remote: str) -> MDRSSession:
config = ConfigFile(remote)
if config.url is None:
raise MissingConfigurationException(f"Remote host `{remote}` is not found.")
return MDRSSession(config.remote, config.url)
def find_laboratory(session: MDRSSession, laboratory_name: str) -> Laboratory:
if session.laboratories.empty():
laboratory_api = LaboratoryApi(session)
session.laboratories = laboratory_api.list()
laboratory = session.laboratories.find_by_name(laboratory_name)
if laboratory is None:
raise IllegalArgumentException(f"Laboratory `{laboratory_name}` not found.")
return laboratory
def find_folder(session: MDRSSession, laboratory: Laboratory, path: str) -> Folder:
folder_api = FolderApi(session)
folders = folder_api.list(laboratory.id, path)
if len(folders) != 1:
raise UnexpectedException(f"Folder `{path}` not found.")
if folders[0].lock:
raise UnauthorizedException(f"Folder `{path}` is locked.")
return folder_api.retrieve(folders[0].id)
def parse_remote_host(path: str) -> str:
path_array = path.split(":")
remote_host = path_array[0]
if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2:
raise IllegalArgumentException("Invalid remote host")
return remote_host
def parse_remote_host_with_path(path: str) -> tuple[str, str, str]:
path = re.sub(r"//+|/\./+|/\.$", "/", path)
if re.search(r"/\.\./|/\.\.$", path) is not None:
raise IllegalArgumentException("Path traversal found.")
path_array = path.split(":")
if len(path_array) != 2:
raise IllegalArgumentException("Invalid remote host.")
remote_host = path_array[0]
folder_array = path_array[1].split("/")
is_absolute_path = folder_array[0] == ""
if not is_absolute_path:
raise IllegalArgumentException("Must be absolute paths.")
del folder_array[0]
if len(folder_array) == 0:
laboratory = ""
folder = ""
else:
laboratory = folder_array.pop(0)
folder = "/" + "/".join(folder_array)
return (remote_host, laboratory, folder)

80
mdrsclient/config.py Normal file
View File

@ -0,0 +1,80 @@
import configparser
import os
from typing import Final
import validators
from mdrsclient.exceptions import IllegalArgumentException
from mdrsclient.settings import CONFIG_FILE_PATH
class ConfigFile:
OPTION_URL: Final[str] = "url"
serial: int
config_file: str
config_dir: str
remote: str
config: configparser.ConfigParser
def __init__(self, remote: str) -> None:
self.serial = -1
self.config_file = CONFIG_FILE_PATH
self.config_dir = os.path.dirname(CONFIG_FILE_PATH)
self.remote = remote
self.config = configparser.ConfigParser()
def list(self) -> list[tuple[str, str]]:
ret: list[tuple[str, str]] = []
self.__load()
for remote in self.config.sections():
url = self.config.get(remote, self.OPTION_URL)
ret.append((remote, url))
return ret
@property
def url(self) -> str | None:
if not self.__exists(self.remote):
return None
return self.config.get(self.remote, self.OPTION_URL)
@url.setter
def url(self, url: str) -> None:
if not validators.url(url): # type: ignore
raise IllegalArgumentException("malformed URI sequence")
self.__load()
if self.config.has_section(self.remote):
self.config.remove_section(self.remote)
self.config.add_section(self.remote)
self.config.set(self.remote, self.OPTION_URL, url)
self.__save()
@url.deleter
def url(self) -> None:
if self.__exists(self.remote):
self.config.remove_section(self.remote)
self.__save()
def __exists(self, section: str) -> bool:
self.__load()
return self.config.has_option(section, self.OPTION_URL)
def __load(self) -> None:
if os.path.isfile(self.config_file):
stat = os.stat(self.config_file)
serial = hash(stat)
if self.serial != serial:
self.config.read(self.config_file, encoding="utf8")
self.serial = serial
def __save(self) -> None:
self.__ensure_cache_dir()
with open(self.config_file, "w") as f:
self.config.write(f)
os.chmod(self.config_file, 0o600)
def __ensure_cache_dir(self) -> None:
if not os.path.exists(self.config_dir):
os.makedirs(self.config_dir)
# ensure directory is secure.
os.chmod(self.config_dir, 0o700)

40
mdrsclient/exceptions.py Normal file
View File

@ -0,0 +1,40 @@
class MDRSException(Exception):
"""Thrown when some kind of errors occurred"""
pass
class IllegalArgumentException(MDRSException):
"""Thrown when something wrong with the argument is passed to a method"""
pass
class MissingConfigurationException(MDRSException):
"""Thrown when wrong or missing system configuration"""
pass
class BadRequestException(MDRSException):
"""Thrown when the request does not contain valid parameter"""
pass
class UnauthorizedException(MDRSException):
"""Thrown when the current user not allowed to perform an operation on the resource"""
pass
class ForbiddenException(MDRSException):
"""Thrown when the current user does not have enough privileges to access the resource"""
pass
class UnexpectedException(MDRSException):
"""Thrown when unexpected error occurred"""
pass

View File

@ -0,0 +1,16 @@
from mdrsclient.models.error import DRFStandardizedErrors
from mdrsclient.models.file import File
from mdrsclient.models.folder import Folder, FolderSimple
from mdrsclient.models.laboratory import Laboratories, Laboratory
from mdrsclient.models.user import Token, User
__all__ = [
"DRFStandardizedErrors",
"File",
"Folder",
"FolderSimple",
"Laboratories",
"Laboratory",
"Token",
"User",
]

View File

@ -0,0 +1,14 @@
from pydantic.dataclasses import dataclass
@dataclass(frozen=True)
class DRFStandardizedError:
code: str
detail: str
attr: str | None
@dataclass(frozen=True)
class DRFStandardizedErrors:
type: str
errors: list[DRFStandardizedError]

25
mdrsclient/models/file.py Normal file
View File

@ -0,0 +1,25 @@
from pydantic.dataclasses import dataclass
from mdrsclient.models.utils import iso8601_to_user_friendly
@dataclass(frozen=True)
class File:
id: str
name: str
type: str
size: int
thumbnail: str | None
description: str
metadata: dict
download_url: str
created_at: str
updated_at: str
@property
def created_at_name(self) -> str:
return iso8601_to_user_friendly(self.created_at)
@property
def updated_at_name(self) -> str:
return iso8601_to_user_friendly(self.updated_at)

View File

@ -0,0 +1,58 @@
from typing import Final
from pydantic.dataclasses import dataclass
from mdrsclient.models import File
from mdrsclient.models.utils import iso8601_to_user_friendly
ACCESS_LEVEL_NAMES: Final[dict[int, str]] = {
-1: "Storage",
0: "Private",
1: "CBS Open",
2: "PW Open",
3: "Public",
}
@dataclass(frozen=True)
class FolderSimple:
id: str
pid: str | None
name: str
access_level: int
lock: bool
lab_id: int
description: str
created_at: str
updated_at: str
restrict_opened_at: str | None
@property
def access_level_name(self) -> str:
return ACCESS_LEVEL_NAMES[self.access_level]
@property
def lock_name(self) -> str:
return "locked" if self.lock else "unlocked"
@property
def created_at_name(self) -> str:
return iso8601_to_user_friendly(self.created_at)
@property
def updated_at_name(self) -> str:
return iso8601_to_user_friendly(self.updated_at)
@dataclass(frozen=True)
class Folder(FolderSimple):
metadata: list[dict]
sub_folders: list[FolderSimple]
files: list[File]
path: str
def find_sub_folder(self, name: str) -> FolderSimple | None:
return next((x for x in self.sub_folders if x.name == name), None)
def find_file(self, name: str) -> File | None:
return next((x for x in self.files if x.name == name), None)

View File

@ -0,0 +1,34 @@
from typing import Generator
from pydantic.dataclasses import dataclass
@dataclass(frozen=True)
class Laboratory:
id: int
name: str
pi_name: str
full_name: str
@dataclass(frozen=True)
class Laboratories:
items: list[Laboratory]
def __iter__(self) -> Generator[Laboratory, None, None]:
yield from self.items
def empty(self) -> bool:
return len(self.items) == 0
def clear(self) -> None:
self.items.clear()
def append(self, item: Laboratory) -> None:
self.items.append(item)
def find_by_id(self, id: int) -> Laboratory | None:
return next((x for x in self.items if x.id == id), None)
def find_by_name(self, name: str) -> Laboratory | None:
return next((x for x in self.items if x.name == name), None)

50
mdrsclient/models/user.py Normal file
View File

@ -0,0 +1,50 @@
import time
import jwt
from pydantic import parse_obj_as
from pydantic.dataclasses import dataclass
@dataclass(frozen=True)
class DecodedJWT:
token_type: str
exp: int
iat: int
jti: str
user_id: int
@dataclass(frozen=True)
class Token:
access: str
refresh: str
@property
def user_id(self) -> int:
access_decoded = self.__decode(self.access)
return access_decoded.user_id
@property
def is_expired(self) -> bool:
now = int(time.time()) + 10
refresh_decoded = self.__decode(self.refresh)
return now > refresh_decoded.exp
@property
def is_refresh_required(self) -> bool:
now = int(time.time()) + 10
access_decoded = self.__decode(self.access)
refresh_decoded = self.__decode(self.refresh)
return now > access_decoded.exp and now < refresh_decoded.exp
def __decode(self, token: str) -> DecodedJWT:
data = jwt.decode(token, options={"verify_signature": False})
return parse_obj_as(DecodedJWT, data)
@dataclass(frozen=True)
class User:
id: int
username: str
laboratory_id: int
laboratory: str

View File

@ -0,0 +1,5 @@
import datetime
def iso8601_to_user_friendly(text: str) -> str:
return datetime.datetime.fromisoformat(text).strftime("%Y/%m/%d %H:%M:%S")

58
mdrsclient/session.py Normal file
View File

@ -0,0 +1,58 @@
import requests
from mdrsclient.cache import CacheFile
from mdrsclient.exceptions import MissingConfigurationException
from mdrsclient.models import Laboratories, Token, User
class MDRSSession(requests.Session):
url: str
__cache: CacheFile
def __init__(self, remote: str, url: str) -> None:
super().__init__()
self.url = url
self.__cache = CacheFile(remote)
self.__prepare_headers()
def build_url(self, *args) -> str:
if self.url == "":
raise MissingConfigurationException("remote host is not configured")
parts = [self.url]
parts.extend(args)
return "/".join(parts)
def logout(self) -> None:
del self.__cache.user
del self.__cache.token
self.headers.update({"Authorization": ""})
@property
def user(self) -> User | None:
return self.__cache.user
@user.setter
def user(self, user: User) -> None:
self.__cache.user = user
@property
def token(self) -> Token | None:
return self.__cache.token
@token.setter
def token(self, token: Token) -> None:
self.__cache.token = token
self.__prepare_headers()
@property
def laboratories(self) -> Laboratories:
return self.__cache.laboratories
@laboratories.setter
def laboratories(self, laboratories: Laboratories) -> None:
self.__cache.laboratories = laboratories
def __prepare_headers(self) -> None:
self.headers.update({"accept": "application/json"})
if self.token is not None:
self.headers.update({"Authorization": f"Bearer {self.token.access}"})

20
mdrsclient/settings.py Normal file
View File

@ -0,0 +1,20 @@
import os
from pydantic import BaseSettings
class Settings(BaseSettings):
config_dir_path: str = "~/.mdrs-client"
number_of_process: int = 10
class Config:
env_file = ".env"
env_file_encoding = "utf-8"
settings = Settings()
NUMBER_OF_PROCESS = settings.number_of_process
CONFIG_DIR_PATH = os.path.expanduser(settings.config_dir_path)
CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, "config.ini")