commit 819c4a6a07338fc164781ff8a397457ab5e81c2c Author: Yoshihiro OKUMURA Date: Mon May 1 20:00:32 2023 +0900 first commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5e8f3ee --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# CONFIG_DIR_PATH="~/.mdrs-client" +# NUMBER_OF_PROCESS=10 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7853142 --- /dev/null +++ b/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# mdrs-cli +.neurodatacli.config \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..588f27f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,14 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.black-formatter", + "ms-python.isort", + "ms-python.flake8", + "ms-python.vscode-pylance", + "esbenp.prettier-vscode", + "redhat.vscode-xml", + "njpwerner.autodocstring", + "mosapride.zenkaku", + "streetsidesoftware.code-spell-checker" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ca3a632 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,38 @@ +{ + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "[xml]": { + "editor.defaultFormatter": "redhat.vscode-xml" + }, + "[jsonc]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + // Extensions - Code Spell Checker + "cSpell.ignoreWords": ["getframe", "pydantic"], + "cSpell.words": ["mdrs", "mdrsclient", "neurodata", "Neuroinformatics", "RIKEN"], + // Extensions - isort + "isort.args": ["--profile=black"], + // Extensions - Prettier + "prettier.printWidth": 120, + "prettier.singleQuote": true, + "prettier.tabWidth": 4, + // Extensions - Pylance + "python.analysis.typeCheckingMode": "basic", + "python.analysis.exclude": ["api/migrations/[0-9]*.py"], + // Extensions - Python:black + "python.formatting.blackArgs": ["--line-length=120"], + "python.formatting.provider": "black", + // Extensions - Python:Flake8 + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=120"], + "python.linting.ignorePatterns": ["**/site-packages/**/*.py", ".vscode/*.py"], + "python.linting.lintOnSave": true, + // Extensions - Python Docstring Generator configuration + "autoDocstring.docstringFormat": "google" +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a57106c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Neuroinformatics Unit, RIKEN CBS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e1d289 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# mdrs-client-python +The mdrs-client-python is python library and a command-line client for up- and downloading files to and from MDRS based repository. + +## Installing +``` +pip install -e . +``` + +## Example Usage + +### config create +Create remote host configuration +``` +$ mdrs config create neurodata https://neurodata.riken.jp/api +``` + +### login +login to remote host +``` +$ mdrs login neurodata: +Username: (enter your login name) +Password: (enter your password) +``` + +### logout +Logout from remote host +``` +$ mdrs logout neurodata: +``` + +### whoami +Print effective user name +``` +$ mdrs whoami neurodata: +``` + +### labs +List all laboratories +``` +$ mdrs labs neurodata: +``` + +### ls +List the folder contents +``` +$ mdrs ls neurodata:/NIU/Repository/ +``` + +### mkdir +Create a new folder +``` +$ mdrs mkdir neurodata:/NIU/Repository/TEST +``` + +### rmdir +Remove a existing folder +``` +$ mdrs rmdir neurodata:/NIU/Repository/TEST +``` + +### metadata +Get a folder metadata +``` +$ mdrs metadata neurodata:/NIU/Repository/TEST +``` + +### upload +Upload the file or directories +``` +$ mdrs upload ./sample.dat neurodata:/NIU/Repository/TEST/ +$ mdrs upload -r ./dataset neurodata:/NIU/Repository/TEST/ +``` + +### download +Download a file +``` +$ mdrs download neurodata:/NIU/Repository/TEST/sample.dat ./ +``` + +### move +Move a file +``` +$ mdrs move neurodata:/NIU/Repository/TEST/sample.dat neurodata:/NIU/Repository/TEST2/sample2.dat +``` + +### remove +Remove a file +``` +$ mdrs remove neurodata:/NIU/Repository/TEST2/sample2.dat +``` + +### file-metadata +Get the file metadata +``` +$ mdrs file-metadata neurodata:/NIU/Repository/TEST/dataset/sample.dat +``` + +### help +Show the help message and exit +``` +$ mdrs -h +``` diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/mdrsclient/__init__.py b/mdrsclient/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mdrsclient/__main__.py b/mdrsclient/__main__.py new file mode 100644 index 0000000..752d94b --- /dev/null +++ b/mdrsclient/__main__.py @@ -0,0 +1,37 @@ +import argparse + +from mdrsclient.commands import ( + ConfigCommand, + FileCommand, + FolderCommand, + LaboratoryCommand, + UserCommand, +) +from mdrsclient.exceptions import MDRSException + + +def main() -> None: + description = """This is a command-line program to up files.""" + + parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawDescriptionHelpFormatter) + subparsers = parser.add_subparsers(title="subcommands") + + ConfigCommand.register(subparsers) + UserCommand.register(subparsers) + LaboratoryCommand.register(subparsers) + FolderCommand.register(subparsers) + FileCommand.register(subparsers) + + try: + args = parser.parse_args() + if hasattr(args, "func"): + args.func(args) + else: + parser.print_help() + except MDRSException as e: + print(f"Error: {e}") + exit(2) + + +if __name__ == "__main__": + main() diff --git a/mdrsclient/__version__.py b/mdrsclient/__version__.py new file mode 100644 index 0000000..daf6a2d --- /dev/null +++ b/mdrsclient/__version__.py @@ -0,0 +1,8 @@ +import os + +here = os.path.abspath(os.path.dirname(__file__)) +__all__ = ["__version__"] + + +with open(os.path.join(os.path.dirname(here), "VERSION")) as version_file: + __version__ = version_file.read().strip() diff --git a/mdrsclient/api/__init__.py b/mdrsclient/api/__init__.py new file mode 100644 index 0000000..27e062c --- /dev/null +++ b/mdrsclient/api/__init__.py @@ -0,0 +1,11 @@ +from mdrsclient.api.file import FileApi +from mdrsclient.api.folder import FolderApi +from mdrsclient.api.laboratory import LaboratoryApi +from mdrsclient.api.user import UserApi + +__all__ = [ + "FileApi", + "FolderApi", + "LaboratoryApi", + "UserApi", +] diff --git a/mdrsclient/api/base.py b/mdrsclient/api/base.py new file mode 100644 index 0000000..a0c71ba --- /dev/null +++ b/mdrsclient/api/base.py @@ -0,0 +1,51 @@ +from abc import ABC + +import requests +from pydantic import parse_obj_as +from requests import Response + +from mdrsclient.exceptions import ( + BadRequestException, + ForbiddenException, + UnauthorizedException, + UnexpectedException, +) +from mdrsclient.models import DRFStandardizedErrors +from mdrsclient.session import MDRSSession + + +class BaseApi(ABC): + def __init__(self, session: MDRSSession) -> None: + self.session = session + + def _get(self, url, *args, **kwargs) -> Response: + return self.session.get(self.__build_url(url), *args, **kwargs) + + def _post(self, url, *args, **kwargs) -> Response: + return self.session.post(self.__build_url(url), *args, **kwargs) + + def _put(self, url, *args, **kwargs) -> Response: + return self.session.put(self.__build_url(url), *args, **kwargs) + + def _delete(self, url, *args, **kwargs) -> Response: + return self.session.delete(self.__build_url(url), *args, **kwargs) + + def _patch(self, url, *args, **kwargs) -> Response: + return self.session.patch(self.__build_url(url), *args, **kwargs) + + def _raise_response_error(self, response: Response) -> None: + if response.status_code >= 300: + if response.status_code < 400 or response.status_code >= 500: + raise UnexpectedException(f"Unexpected status code returned: {response.status_code}.") + errors = parse_obj_as(DRFStandardizedErrors, response.json()) + if response.status_code == requests.codes.bad_request: + raise BadRequestException(errors.errors[0].detail) + elif response.status_code == requests.codes.unauthorized: + raise UnauthorizedException("Login required.") + elif response.status_code == requests.codes.forbidden: + raise ForbiddenException("You do not have enough permissions. Access is denied.") + else: + raise UnexpectedException(errors.errors[0].detail) + + def __build_url(self, *args: tuple) -> str: + return self.session.build_url(*args) diff --git a/mdrsclient/api/file.py b/mdrsclient/api/file.py new file mode 100644 index 0000000..606876a --- /dev/null +++ b/mdrsclient/api/file.py @@ -0,0 +1,82 @@ +import sys +from typing import Final + +from pydantic import parse_obj_as +from pydantic.dataclasses import dataclass + +from mdrsclient.api.base import BaseApi +from mdrsclient.api.utils import token_check +from mdrsclient.exceptions import UnexpectedException +from mdrsclient.models import File + + +@dataclass(frozen=True) +class FileCreateResponse: + id: str + + +class FileApi(BaseApi): + ENTRYPOINT: Final[str] = "v2/file/" + + def retrieve(self, id: str) -> File: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + id + "/" + token_check(self.session) + response = self._get(url) + self._raise_response_error(response) + return parse_obj_as(File, response.json()) + + def create(self, folder_id: str, path: str) -> str: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + token_check(self.session) + data = {"folder_id": folder_id} + try: + with open(path, mode="rb") as fp: + response = self._post(url, data=data, files={"file": fp}) + self._raise_response_error(response) + ret = parse_obj_as(FileCreateResponse, response.json()) + except OSError: + raise UnexpectedException(f"Could not open `{path}` file.") + return ret.id + + def update(self, file: File, path: str | None) -> bool: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + file.id + "/" + token_check(self.session) + if path is not None: + try: + with open(path, mode="rb") as fp: + response = self._put(url, files={"file": fp}) + except OSError: + raise UnexpectedException(f"Could not open `{path}` file.") + else: + data = {"name": file.name, "description": file.description} + response = self._put(url, data=data) + self._raise_response_error(response) + return True + + def destroy(self, file: File) -> bool: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + file.id + "/" + token_check(self.session) + response = self._delete(url) + self._raise_response_error(response) + return True + + def move(self, file: File, folder_id: str | None) -> bool: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + file.id + "/move/" + data = {"folder": folder_id} + token_check(self.session) + response = self._post(url, data=data) + self._raise_response_error(response) + return True + + def metadata(self, file: File) -> dict: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + file.id + "/metadata/" + token_check(self.session) + response = self._get(url) + self._raise_response_error(response) + return response.json() diff --git a/mdrsclient/api/folder.py b/mdrsclient/api/folder.py new file mode 100644 index 0000000..526359c --- /dev/null +++ b/mdrsclient/api/folder.py @@ -0,0 +1,77 @@ +import sys +from typing import Final + +from pydantic import parse_obj_as +from pydantic.dataclasses import dataclass + +from mdrsclient.api.base import BaseApi +from mdrsclient.api.utils import token_check +from mdrsclient.models import Folder, FolderSimple + + +@dataclass(frozen=True) +class FolderCreateResponse: + id: str + + +class FolderApi(BaseApi): + ENTRYPOINT: Final[str] = "v2/folder/" + + def list(self, laboratory_id: int, path: str) -> list[FolderSimple]: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + params = {"path": path, "laboratory_id": laboratory_id} + token_check(self.session) + response = self._get(url, params=params) + self._raise_response_error(response) + ret: list[FolderSimple] = [] + for data in response.json(): + ret.append(parse_obj_as(FolderSimple, data)) + return ret + + def retrieve(self, id: str) -> Folder: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + id + "/" + token_check(self.session) + response = self._get(url) + self._raise_response_error(response) + ret = parse_obj_as(Folder, response.json()) + return ret + + def create(self, name: str, parent_id: str) -> str: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + data = {"name": name, "parent_id": parent_id, "description": "", "template_id": -1} + token_check(self.session) + response = self._post(url, data=data) + self._raise_response_error(response) + ret = parse_obj_as(FolderCreateResponse, response.json()) + return ret.id + + def update(self, folder: FolderSimple) -> bool: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + data = { + "name": folder.name, + "description": folder.description, + } + token_check(self.session) + response = self._put(url, data=data) + self._raise_response_error(response) + return True + + def destroy(self, id: str) -> bool: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + id + "/" + token_check(self.session) + response = self._delete(url) + self._raise_response_error(response) + return True + + def metadata(self, id: str) -> dict: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + id + "/metadata/" + token_check(self.session) + response = self._get(url) + self._raise_response_error(response) + return response.json() diff --git a/mdrsclient/api/laboratory.py b/mdrsclient/api/laboratory.py new file mode 100644 index 0000000..372341c --- /dev/null +++ b/mdrsclient/api/laboratory.py @@ -0,0 +1,23 @@ +import sys +from typing import Final + +from pydantic import parse_obj_as + +from mdrsclient.api.base import BaseApi +from mdrsclient.api.utils import token_check +from mdrsclient.models import Laboratories, Laboratory + + +class LaboratoryApi(BaseApi): + ENTRYPOINT: Final[str] = "v2/laboratory/" + + def list(self) -> Laboratories: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + token_check(self.session) + response = self._get(url) + self._raise_response_error(response) + ret = Laboratories([]) + for data in response.json(): + ret.append(parse_obj_as(Laboratory, data)) + return ret diff --git a/mdrsclient/api/user.py b/mdrsclient/api/user.py new file mode 100644 index 0000000..0ab1eaf --- /dev/null +++ b/mdrsclient/api/user.py @@ -0,0 +1,44 @@ +import sys +from typing import Final + +import requests +from pydantic import parse_obj_as +from pydantic.dataclasses import dataclass + +from mdrsclient.api.base import BaseApi +from mdrsclient.exceptions import UnauthorizedException +from mdrsclient.models import Token, User + + +@dataclass(frozen=True) +class UserAuthResponse(Token): + laboratory: str + lab_id: int + + +class UserApi(BaseApi): + ENTRYPOINT: Final[str] = "v2/" + + def auth(self, username: str, password: str) -> tuple[User, Token]: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + "auth/" + data = {"username": username, "password": password} + response = self._post(url, data=data) + if response.status_code == requests.codes.unauthorized: + raise UnauthorizedException("Invalid username or password.") + self._raise_response_error(response) + obj = parse_obj_as(UserAuthResponse, response.json()) + token = Token(access=obj.access, refresh=obj.refresh) + user = User(id=token.user_id, username=username, laboratory_id=obj.lab_id, laboratory=obj.laboratory) + return (user, token) + + def refresh(self, token: Token) -> Token: + print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name) + url = self.ENTRYPOINT + "refresh/" + data = {"refresh": token.refresh} + response = self._post(url, data=data) + if response.status_code == requests.codes.unauthorized: + raise UnauthorizedException("Token is invalid or expired.") + self._raise_response_error(response) + token = parse_obj_as(Token, response.json()) + return token diff --git a/mdrsclient/api/utils.py b/mdrsclient/api/utils.py new file mode 100644 index 0000000..d699cdf --- /dev/null +++ b/mdrsclient/api/utils.py @@ -0,0 +1,15 @@ +from mdrsclient.api.user import UserApi +from mdrsclient.exceptions import UnauthorizedException +from mdrsclient.session import MDRSSession + + +def token_check(session: MDRSSession) -> None: + if session.token is not None: + if session.token.is_refresh_required: + user_api = UserApi(session) + try: + session.token = user_api.refresh(session.token) + except UnauthorizedException: + session.logout() + elif session.token.is_expired: + session.logout() diff --git a/mdrsclient/cache.py b/mdrsclient/cache.py new file mode 100644 index 0000000..26645c8 --- /dev/null +++ b/mdrsclient/cache.py @@ -0,0 +1,113 @@ +import dataclasses +import fcntl +import json +import os + +from pydantic import ValidationError +from pydantic.dataclasses import dataclass +from pydantic.tools import parse_obj_as + +from mdrsclient.models import Laboratories, Token, User +from mdrsclient.settings import CONFIG_DIR_PATH + + +@dataclass +class CacheData: + user: User | None + token: Token | None + laboratories: Laboratories + + +class CacheFile: + serial: int + cache_dir: str + cache_file: str + data: CacheData + + def __init__(self, remote: str) -> None: + self.serial = -1 + self.cache_dir = os.path.join(CONFIG_DIR_PATH, "cache") + self.cache_file = os.path.join(self.cache_dir, remote + ".json") + self.data = CacheData(user=None, token=None, laboratories=Laboratories([])) + + def dump(self) -> CacheData | None: + self.__load() + return self.data + + @property + def token(self) -> Token | None: + self.__load() + return self.data.token + + @token.setter + def token(self, token: Token) -> None: + self.__load() + self.data.token = token + self.__save() + + @token.deleter + def token(self) -> None: + if self.data.token is not None: + self.__clear() + + @property + def user(self) -> User | None: + return self.data.user + + @user.setter + def user(self, user: User) -> None: + self.__load() + self.data.user = user + self.__save() + + @user.deleter + def user(self) -> None: + if self.data.user is not None: + self.__clear() + + @property + def laboratories(self) -> Laboratories: + return self.data.laboratories + + @laboratories.setter + def laboratories(self, laboratories: Laboratories) -> None: + self.__load() + self.data.laboratories = laboratories + self.__save() + + def __clear(self) -> None: + self.data.user = None + self.data.token = None + self.data.laboratories.clear() + self.__save() + + def __load(self) -> None: + if os.path.isfile(self.cache_file): + stat = os.stat(self.cache_file) + serial = hash((stat.st_uid, stat.st_gid, stat.st_mode, stat.st_size, stat.st_mtime)) + if self.serial != serial: + try: + with open(self.cache_file) as f: + self.data = parse_obj_as(CacheData, json.load(f)) + except ValidationError: + self.__clear() + self.__save() + else: + self.serial = serial + else: + self.data.token = None + self.serial = -1 + + def __save(self) -> None: + self.__ensure_cache_dir() + with open(self.cache_file, "w") as f: + fcntl.flock(f, fcntl.LOCK_EX) + f.write(json.dumps(dataclasses.asdict(self.data))) + # ensure file is secure. + os.chmod(self.cache_file, 0o600) + + def __ensure_cache_dir(self) -> None: + if not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + # ensure directory is secure. + os.chmod(self.cache_dir, 0o700) diff --git a/mdrsclient/commands/__init__.py b/mdrsclient/commands/__init__.py new file mode 100644 index 0000000..7013a53 --- /dev/null +++ b/mdrsclient/commands/__init__.py @@ -0,0 +1,13 @@ +from mdrsclient.commands.config import ConfigCommand +from mdrsclient.commands.file import FileCommand +from mdrsclient.commands.folder import FolderCommand +from mdrsclient.commands.laboratory import LaboratoryCommand +from mdrsclient.commands.user import UserCommand + +__all__ = [ + "ConfigCommand", + "FileCommand", + "FolderCommand", + "LaboratoryCommand", + "UserCommand", +] diff --git a/mdrsclient/commands/base.py b/mdrsclient/commands/base.py new file mode 100644 index 0000000..1442a62 --- /dev/null +++ b/mdrsclient/commands/base.py @@ -0,0 +1,11 @@ +from abc import ABC, abstractmethod +from argparse import _SubParsersAction + +from mdrsclient.exceptions import UnexpectedException + + +class BaseCommand(ABC): + @staticmethod + @abstractmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + raise UnexpectedException("Not implemented.") diff --git a/mdrsclient/commands/config.py b/mdrsclient/commands/config.py new file mode 100644 index 0000000..84442a2 --- /dev/null +++ b/mdrsclient/commands/config.py @@ -0,0 +1,69 @@ +from argparse import Namespace, _SubParsersAction + +from mdrsclient.commands.base import BaseCommand +from mdrsclient.commands.utils import parse_remote_host +from mdrsclient.config import ConfigFile +from mdrsclient.exceptions import IllegalArgumentException + + +class ConfigCommand(BaseCommand): + @staticmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + # config + parser = top_level_subparsers.add_parser("config", help="configure remote hosts") + parser.set_defaults(func=lambda x: parser.print_help()) + subparsers = parser.add_subparsers(title="config subcommands") + # config create + create_parser = subparsers.add_parser("create", help="create a new remote host") + create_parser.add_argument("remote", help="Label of remote host") + create_parser.add_argument("url", help="API entrypoint url of remote host") + create_parser.set_defaults(func=ConfigCommand.create) + # config update + update_parser = subparsers.add_parser("update", help="update a new remote host") + update_parser.add_argument("remote", help="Label of remote host") + update_parser.add_argument("url", help="API entrypoint url of remote host") + update_parser.set_defaults(func=ConfigCommand.update) + # config list + list_parser = subparsers.add_parser("list", help="list all the remote hosts") + list_parser.add_argument("-l", "--long", help="Show the api url", action="store_true") + list_parser.set_defaults(func=ConfigCommand.list) + # config delete + delete_parser = subparsers.add_parser("delete", help="delete an existing remote host") + delete_parser.add_argument("remote", help="Label of remote host") + delete_parser.set_defaults(func=ConfigCommand.delete) + + @staticmethod + def create(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote=remote) + if config.url is not None: + raise IllegalArgumentException(f"Remote host `{remote}` is already exists.") + else: + config.url = args.url + + @staticmethod + def update(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote=remote) + if config.url is None: + raise IllegalArgumentException(f"Remote host `{remote}` is not exists.") + else: + config.url = args.url + + @staticmethod + def list(args: Namespace) -> None: + config = ConfigFile("") + for remote, url in config.list(): + line = f"{remote}:" + if args.long: + line += f"\t{url}" + print(line) + + @staticmethod + def delete(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote=remote) + if config.url is None: + raise IllegalArgumentException(f"Remote host `{remote}` is not exists.") + else: + del config.url diff --git a/mdrsclient/commands/file.py b/mdrsclient/commands/file.py new file mode 100644 index 0000000..d812b33 --- /dev/null +++ b/mdrsclient/commands/file.py @@ -0,0 +1,231 @@ +import dataclasses +import os +from argparse import Namespace, _SubParsersAction +from multiprocessing import Process + +from pydantic import parse_obj_as +from pydantic.dataclasses import dataclass + +from mdrsclient.api import FileApi, FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.commands.utils import ( + create_session, + find_folder, + find_laboratory, + parse_remote_host_with_path, +) +from mdrsclient.exceptions import ( + IllegalArgumentException, + MDRSException, + UnexpectedException, +) +from mdrsclient.models import File, Folder +from mdrsclient.session import MDRSSession +from mdrsclient.settings import NUMBER_OF_PROCESS + + +@dataclass(frozen=True) +class UploadFile: + folder: Folder + path: str + + +class FileCommand(BaseCommand): + @staticmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + # upload + upload_parser = top_level_subparsers.add_parser("upload", help="upload the file or directories") + upload_parser.add_argument( + "-r", "--recursive", help="Upload directories and their contents recursive", action="store_true" + ) + upload_parser.add_argument("local_path", help="Local file path (/foo/bar/data.txt)") + upload_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") + upload_parser.set_defaults(func=FileCommand.upload) + # download + download_parser = top_level_subparsers.add_parser("download", help="download a file") + download_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") + download_parser.add_argument("local_path", help="Local folder path (/foo/bar/)") + download_parser.set_defaults(func=FileCommand.download) + # move + move_parser = top_level_subparsers.add_parser("move", help="move a file") + move_parser.add_argument("src_path", help="Source remote file path (remote:/lab/path/file)") + move_parser.add_argument("dest_path", help="Destination remote file path (remote:/lab/path/file)") + move_parser.set_defaults(func=FileCommand.move) + # remove + remove_parser = top_level_subparsers.add_parser("remove", help="remove a file") + remove_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") + remove_parser.set_defaults(func=FileCommand.remove) + # file-metadata + metadata_parser = top_level_subparsers.add_parser("file-metadata", help="get the file metadata") + metadata_parser.add_argument("remote_path", help="Remote file path (remote:/lab/path/file)") + metadata_parser.set_defaults(func=FileCommand.metadata) + + @staticmethod + def upload(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + local_path = os.path.realpath(args.local_path) + if not os.path.exists(local_path): + raise IllegalArgumentException(f"File or directory `{args.local_path}` not found.") + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, path) + upload_files: list[UploadFile] = [] + if os.path.isdir(local_path): + if not args.recursive: + raise IllegalArgumentException(f"Cannot upload `{args.local_path}`: Is a directory.") + folder_api = FolderApi(session) + folders: dict[str, Folder] = {} + folders[path] = folder + local_basename = os.path.basename(local_path) + for dirpath, dirnames, filenames in os.walk(local_path): + sub = ( + local_basename + if dirpath == local_path + else os.path.join(local_basename, os.path.relpath(dirpath, local_path)) + ) + dest_folder_path = os.path.join(path, sub) + dest_folder_name = os.path.basename(dest_folder_path) + # prepare destination parent path + dest_parent_folder_path = os.path.dirname(dest_folder_path) + if folders.get(dest_parent_folder_path) is None: + res = folder_api.list(laboratory.id, dest_parent_folder_path) + if len(res) != 1: + raise UnexpectedException(f"Remote folder `{dest_parent_folder_path}` not found.") + folders[dest_parent_folder_path] = folder_api.retrieve(res[0].id) + # prepare destination path + if folders.get(dest_folder_path) is None: + dest_folder_simple = folders[dest_parent_folder_path].find_sub_folder(dest_folder_name) + if dest_folder_simple is None: + dest_folder_id = folder_api.create(dest_folder_name, folders[dest_parent_folder_path].id) + else: + dest_folder_id = dest_folder_simple.id + folders[dest_folder_path] = folder_api.retrieve(dest_folder_id) + if dest_folder_simple is None: + folders[dest_parent_folder_path].sub_folders.append(folders[dest_folder_path]) + # register upload file list + for filename in filenames: + upload_files.append(UploadFile(folders[dest_folder_path], os.path.join(dirpath, filename))) + else: + upload_files.append(UploadFile(folder, local_path)) + FileCommand._multiple_upload(session, upload_files) + + @staticmethod + def download(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + path = path.rstrip("/") + parent_path = os.path.dirname(path) + file_name = os.path.basename(path) + session = create_session(remote) + if not os.path.isdir(args.local_path): + raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.") + local_file = os.path.join(args.local_path, file_name) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, parent_path) + file = folder.find_file(file_name) + if file is None: + raise IllegalArgumentException(f"File `{file_name}` not found.") + r = session.get(session.build_url("v2/" + file.download_url), stream=True) + try: + with open(local_file, "wb") as f: + for chunk in r.iter_content(chunk_size=4096): + if chunk: + f.write(chunk) + f.flush() + except PermissionError: + raise IllegalArgumentException(f"Cannot create file `{local_file}`: Permission denied.") + + @staticmethod + def move(args: Namespace) -> None: + (src_remote, src_laboratory_name, src_path) = parse_remote_host_with_path(args.src_path) + (dest_remote, dest_laboratory_name, dest_path) = parse_remote_host_with_path(args.dest_path) + if src_remote != dest_remote: + raise IllegalArgumentException("Remote host mismatched.") + if src_laboratory_name != dest_laboratory_name: + raise IllegalArgumentException("Laboratory mismatched.") + src_path = src_path.rstrip("/") + src_dirpath = os.path.dirname(src_path) + src_filename = os.path.basename(src_path) + if dest_path.endswith("/"): + dest_dirpath = dest_path + dest_filename = src_filename + else: + dest_dirpath = os.path.dirname(dest_path) + dest_filename = os.path.basename(dest_path) + session = create_session(src_remote) + laboratory = find_laboratory(session, src_laboratory_name) + src_folder = find_folder(session, laboratory, src_dirpath) + dest_folder = find_folder(session, laboratory, dest_dirpath) + src_file = src_folder.find_file(src_filename) + if src_file is None: + raise IllegalArgumentException(f"File `{src_filename}` not found.") + dest_file = dest_folder.find_file(dest_filename) + if dest_file is not None: + raise IllegalArgumentException(f"File `{dest_filename}` already exists.") + file_api = FileApi(session) + if src_folder.id != dest_folder.id: + file_api.move(src_file, dest_folder.id) + if dest_filename != src_filename: + dest_file_dict = dataclasses.asdict(src_file) | {"name": dest_filename} + dest_file = parse_obj_as(File, dest_file_dict) + file_api.update(dest_file, None) + + @staticmethod + def remove(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + path = path.rstrip("/") + parent_path = os.path.dirname(path) + file_name = os.path.basename(path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, parent_path) + file = folder.find_file(file_name) + if file is None: + raise IllegalArgumentException(f"File `{file_name}` not found.") + file_api = FileApi(session) + file_api.destroy(file) + + @staticmethod + def metadata(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + path = path.rstrip("/") + parent_path = os.path.dirname(path) + file_name = os.path.basename(path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, parent_path) + file = folder.find_file(file_name) + if file is None: + raise IllegalArgumentException(f"File `{file_name}` not found.") + file_api = FileApi(session) + metadata = file_api.metadata(file) + print(metadata) + + @staticmethod + def _multiple_upload(session: MDRSSession, upload_files: list[UploadFile]) -> None: + processes: list[Process] = [] + for idx in range(NUMBER_OF_PROCESS): + processes.append( + Process( + target=FileCommand._multiple_upload_worker, + args=(session, upload_files, idx, NUMBER_OF_PROCESS), + ) + ) + for process in processes: + process.start() + for process in processes: + process.join() + + @staticmethod + def _multiple_upload_worker(session: MDRSSession, upload_files: list[UploadFile], idx: int, num_proc: int) -> None: + file_api = FileApi(session) + for upload_file in upload_files[idx::num_proc]: + file_name = os.path.basename(upload_file.path) + file = next((x for x in upload_file.folder.files if x.name == file_name), None) + try: + if file is None: + file_api.create(upload_file.folder.id, upload_file.path) + else: + file_api.update(file, upload_file.path) + pass + except MDRSException as e: + print(f"API Error: {e}") diff --git a/mdrsclient/commands/folder.py b/mdrsclient/commands/folder.py new file mode 100644 index 0000000..26b1f9d --- /dev/null +++ b/mdrsclient/commands/folder.py @@ -0,0 +1,117 @@ +import os +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import FolderApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.commands.utils import ( + create_session, + find_folder, + find_laboratory, + parse_remote_host_with_path, +) + + +class FolderCommand(BaseCommand): + @staticmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + # ls + ls_parser = top_level_subparsers.add_parser("ls", help="list the folder contents") + ls_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") + ls_parser.set_defaults(func=FolderCommand.list) + # mkdir + mkdir_parser = top_level_subparsers.add_parser("mkdir", help="create a new folder") + mkdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") + mkdir_parser.set_defaults(func=FolderCommand.mkdir) + # rmdir + rmdir_parser = top_level_subparsers.add_parser("rmdir", help="remove a existing folder") + rmdir_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") + rmdir_parser.set_defaults(func=FolderCommand.rmdir) + # metadata + metadata_parser = top_level_subparsers.add_parser("metadata", help="get a folder metadata") + metadata_parser.add_argument("remote_path", help="Remote folder path (remote:/lab/path/)") + metadata_parser.set_defaults(func=FolderCommand.metadata) + + @staticmethod + def list(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, path) + label = { + "type": "Type", + "acl": "Access", + "laboratory": "Laboratory", + "size": "Lock/Size", + "date": "Date", + "name": "Name", + } + length: dict[str, int] = {} + for key in label.keys(): + length[key] = len(label[key]) + for sub_folder in folder.sub_folders: + sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id) + sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)" + length["acl"] = max(length["acl"], len(sub_folder.access_level_name)) + length["laboratory"] = max(length["laboratory"], len(sub_laboratory_name)) + length["size"] = max(length["size"], len(sub_folder.lock_name)) + length["date"] = max(length["date"], len(sub_folder.updated_at_name)) + length["name"] = max(length["name"], len(sub_folder.name)) + for file in folder.files: + length["size"] = max(length["size"], len(str(file.size))) + length["date"] = max(length["date"], len(file.updated_at_name)) + length["name"] = max(length["name"], len(file.name)) + length["acl"] = max(length["acl"], len(folder.access_level_name)) + length["laboratory"] = max(length["laboratory"], len(laboratory.name)) + header = ( + f"{label['type']:{length['type']}}\t{label['acl']:{length['acl']}}\t" + f"{label['laboratory']:{length['laboratory']}}\t{label['size']:{length['size']}}\t" + f"{label['date']:{length['date']}}\t{label['name']:{length['name']}}" + ) + print(header) + print("-" * len(header.expandtabs())) + + for sub_folder in sorted(folder.sub_folders, key=lambda x: x.name): + sub_laboratory = session.laboratories.find_by_id(sub_folder.lab_id) + sub_laboratory_name = sub_laboratory.name if sub_laboratory is not None else "(invalid)" + print( + f"{'[d]':{length['type']}}\t{sub_folder.access_level_name:{length['acl']}}\t" + f"{sub_laboratory_name:{length['laboratory']}}\t{sub_folder.lock_name:{length['size']}}\t" + f"{sub_folder.updated_at_name:{length['date']}}\t{sub_folder.name:{length['name']}}" + ) + for file in sorted(folder.files, key=lambda x: x.name): + print( + f"{'[f]':{length['type']}}\t{folder.access_level_name:{length['acl']}}\t" + f"{laboratory.name:{length['laboratory']}}\t{file.size:{length['size']}}\t" + f"{file.updated_at_name:{length['date']}}\t{file.name:{length['name']}}" + ) + + @staticmethod + def mkdir(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + path = path.rstrip("/") + parent_path = os.path.dirname(path) + folder_name = os.path.basename(path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, parent_path) + folder_api = FolderApi(session) + folder_api.create(folder_name, folder.id) + + @staticmethod + def rmdir(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, path) + folder_api = FolderApi(session) + folder_api.destroy(folder.id) + + @staticmethod + def metadata(args: Namespace) -> None: + (remote, laboratory_name, path) = parse_remote_host_with_path(args.remote_path) + session = create_session(remote) + laboratory = find_laboratory(session, laboratory_name) + folder = find_folder(session, laboratory, path) + folder_api = FolderApi(session) + metadata = folder_api.metadata(folder.id) + print(metadata) diff --git a/mdrsclient/commands/laboratory.py b/mdrsclient/commands/laboratory.py new file mode 100644 index 0000000..0fa6536 --- /dev/null +++ b/mdrsclient/commands/laboratory.py @@ -0,0 +1,42 @@ +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import LaboratoryApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.commands.utils import create_session, parse_remote_host + + +class LaboratoryCommand(BaseCommand): + @staticmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + # labs + lls_parser = top_level_subparsers.add_parser("labs", help="list all laboratories") + lls_parser.add_argument("remote", help="Label of remote host") + lls_parser.set_defaults(func=LaboratoryCommand.list) + + @staticmethod + def list(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + session = create_session(remote) + laboratory_api = LaboratoryApi(session) + laboratories = laboratory_api.list() + session.laboratories = laboratories + label = {"id": "ID", "name": "Name", "pi_name": "PI", "full_name": "Laboratory"} + length: dict[str, int] = {} + for key in label.keys(): + length[key] = len(label[key]) + for laboratory in laboratories: + length["id"] = max(length["id"], len(str(laboratory.id))) + length["name"] = max(length["name"], len(laboratory.name)) + length["pi_name"] = max(length["pi_name"], len(laboratory.pi_name)) + length["full_name"] = max(length["full_name"], len(laboratory.full_name)) + header = ( + f"{label['id']:{length['id']}}\t{label['name']:{length['name']}}\t" + f"{label['pi_name']:{length['pi_name']}}\t{label['full_name']:{length['full_name']}}" + ) + print(header) + print("-" * len(header.expandtabs())) + for laboratory in laboratories: + print( + f"{laboratory.id:{length['id']}}\t{laboratory.name:{length['name']}}\t" + f"{laboratory.pi_name:{length['pi_name']}}\t{laboratory.full_name:{length['full_name']}}" + ) diff --git a/mdrsclient/commands/user.py b/mdrsclient/commands/user.py new file mode 100644 index 0000000..26d2680 --- /dev/null +++ b/mdrsclient/commands/user.py @@ -0,0 +1,61 @@ +import getpass +from argparse import Namespace, _SubParsersAction + +from mdrsclient.api import UserApi +from mdrsclient.commands.base import BaseCommand +from mdrsclient.commands.utils import parse_remote_host +from mdrsclient.config import ConfigFile +from mdrsclient.exceptions import MissingConfigurationException +from mdrsclient.session import MDRSSession + + +class UserCommand(BaseCommand): + @staticmethod + def register(top_level_subparsers: _SubParsersAction) -> None: + # login + login_parser = top_level_subparsers.add_parser("login", help="login to remote host") + login_parser.add_argument("remote", help="Label of remote host") + login_parser.set_defaults(func=UserCommand.login) + # logout + logout_parser = top_level_subparsers.add_parser("logout", help="logout from remote host") + logout_parser.add_argument("remote", help="Label of remote host") + logout_parser.set_defaults(func=UserCommand.logout) + # whoami + whoami_parser = top_level_subparsers.add_parser("whoami", help="show current user name") + whoami_parser.add_argument("remote", help="Label of remote host") + whoami_parser.set_defaults(func=UserCommand.whoami) + + @staticmethod + def login(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + session = MDRSSession(config.remote, config.url) + username = input("Username: ").strip() + password = getpass.getpass("Password: ").strip() + user_api = UserApi(session) + (user, token) = user_api.auth(username, password) + session.user = user + session.token = token + + @staticmethod + def logout(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + session = MDRSSession(config.remote, config.url) + session.logout() + + @staticmethod + def whoami(args: Namespace) -> None: + remote = parse_remote_host(args.remote) + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + session = MDRSSession(config.remote, config.url) + if session.token is not None and session.token.is_expired: + session.logout() + username = session.user.username if session.user is not None else "(Anonymous)" + print(username) diff --git a/mdrsclient/commands/utils.py b/mdrsclient/commands/utils.py new file mode 100644 index 0000000..7306ffc --- /dev/null +++ b/mdrsclient/commands/utils.py @@ -0,0 +1,69 @@ +import re + +from mdrsclient.api import FolderApi, LaboratoryApi +from mdrsclient.config import ConfigFile +from mdrsclient.exceptions import ( + IllegalArgumentException, + MissingConfigurationException, + UnauthorizedException, + UnexpectedException, +) +from mdrsclient.models import Folder, Laboratory +from mdrsclient.session import MDRSSession + + +def create_session(remote: str) -> MDRSSession: + config = ConfigFile(remote) + if config.url is None: + raise MissingConfigurationException(f"Remote host `{remote}` is not found.") + return MDRSSession(config.remote, config.url) + + +def find_laboratory(session: MDRSSession, laboratory_name: str) -> Laboratory: + if session.laboratories.empty(): + laboratory_api = LaboratoryApi(session) + session.laboratories = laboratory_api.list() + laboratory = session.laboratories.find_by_name(laboratory_name) + if laboratory is None: + raise IllegalArgumentException(f"Laboratory `{laboratory_name}` not found.") + return laboratory + + +def find_folder(session: MDRSSession, laboratory: Laboratory, path: str) -> Folder: + folder_api = FolderApi(session) + folders = folder_api.list(laboratory.id, path) + if len(folders) != 1: + raise UnexpectedException(f"Folder `{path}` not found.") + if folders[0].lock: + raise UnauthorizedException(f"Folder `{path}` is locked.") + return folder_api.retrieve(folders[0].id) + + +def parse_remote_host(path: str) -> str: + path_array = path.split(":") + remote_host = path_array[0] + if len(path_array) == 2 and path_array[1] != "" or len(path_array) > 2: + raise IllegalArgumentException("Invalid remote host") + return remote_host + + +def parse_remote_host_with_path(path: str) -> tuple[str, str, str]: + path = re.sub(r"//+|/\./+|/\.$", "/", path) + if re.search(r"/\.\./|/\.\.$", path) is not None: + raise IllegalArgumentException("Path traversal found.") + path_array = path.split(":") + if len(path_array) != 2: + raise IllegalArgumentException("Invalid remote host.") + remote_host = path_array[0] + folder_array = path_array[1].split("/") + is_absolute_path = folder_array[0] == "" + if not is_absolute_path: + raise IllegalArgumentException("Must be absolute paths.") + del folder_array[0] + if len(folder_array) == 0: + laboratory = "" + folder = "" + else: + laboratory = folder_array.pop(0) + folder = "/" + "/".join(folder_array) + return (remote_host, laboratory, folder) diff --git a/mdrsclient/config.py b/mdrsclient/config.py new file mode 100644 index 0000000..b69b24c --- /dev/null +++ b/mdrsclient/config.py @@ -0,0 +1,80 @@ +import configparser +import os +from typing import Final + +import validators + +from mdrsclient.exceptions import IllegalArgumentException +from mdrsclient.settings import CONFIG_FILE_PATH + + +class ConfigFile: + OPTION_URL: Final[str] = "url" + + serial: int + config_file: str + config_dir: str + remote: str + config: configparser.ConfigParser + + def __init__(self, remote: str) -> None: + self.serial = -1 + self.config_file = CONFIG_FILE_PATH + self.config_dir = os.path.dirname(CONFIG_FILE_PATH) + self.remote = remote + self.config = configparser.ConfigParser() + + def list(self) -> list[tuple[str, str]]: + ret: list[tuple[str, str]] = [] + self.__load() + for remote in self.config.sections(): + url = self.config.get(remote, self.OPTION_URL) + ret.append((remote, url)) + return ret + + @property + def url(self) -> str | None: + if not self.__exists(self.remote): + return None + return self.config.get(self.remote, self.OPTION_URL) + + @url.setter + def url(self, url: str) -> None: + if not validators.url(url): # type: ignore + raise IllegalArgumentException("malformed URI sequence") + self.__load() + if self.config.has_section(self.remote): + self.config.remove_section(self.remote) + self.config.add_section(self.remote) + self.config.set(self.remote, self.OPTION_URL, url) + self.__save() + + @url.deleter + def url(self) -> None: + if self.__exists(self.remote): + self.config.remove_section(self.remote) + self.__save() + + def __exists(self, section: str) -> bool: + self.__load() + return self.config.has_option(section, self.OPTION_URL) + + def __load(self) -> None: + if os.path.isfile(self.config_file): + stat = os.stat(self.config_file) + serial = hash(stat) + if self.serial != serial: + self.config.read(self.config_file, encoding="utf8") + self.serial = serial + + def __save(self) -> None: + self.__ensure_cache_dir() + with open(self.config_file, "w") as f: + self.config.write(f) + os.chmod(self.config_file, 0o600) + + def __ensure_cache_dir(self) -> None: + if not os.path.exists(self.config_dir): + os.makedirs(self.config_dir) + # ensure directory is secure. + os.chmod(self.config_dir, 0o700) diff --git a/mdrsclient/exceptions.py b/mdrsclient/exceptions.py new file mode 100644 index 0000000..f0b9c98 --- /dev/null +++ b/mdrsclient/exceptions.py @@ -0,0 +1,40 @@ +class MDRSException(Exception): + """Thrown when some kind of errors occurred""" + + pass + + +class IllegalArgumentException(MDRSException): + """Thrown when something wrong with the argument is passed to a method""" + + pass + + +class MissingConfigurationException(MDRSException): + """Thrown when wrong or missing system configuration""" + + pass + + +class BadRequestException(MDRSException): + """Thrown when the request does not contain valid parameter""" + + pass + + +class UnauthorizedException(MDRSException): + """Thrown when the current user not allowed to perform an operation on the resource""" + + pass + + +class ForbiddenException(MDRSException): + """Thrown when the current user does not have enough privileges to access the resource""" + + pass + + +class UnexpectedException(MDRSException): + """Thrown when unexpected error occurred""" + + pass diff --git a/mdrsclient/models/__init__.py b/mdrsclient/models/__init__.py new file mode 100644 index 0000000..dcc9c74 --- /dev/null +++ b/mdrsclient/models/__init__.py @@ -0,0 +1,16 @@ +from mdrsclient.models.error import DRFStandardizedErrors +from mdrsclient.models.file import File +from mdrsclient.models.folder import Folder, FolderSimple +from mdrsclient.models.laboratory import Laboratories, Laboratory +from mdrsclient.models.user import Token, User + +__all__ = [ + "DRFStandardizedErrors", + "File", + "Folder", + "FolderSimple", + "Laboratories", + "Laboratory", + "Token", + "User", +] diff --git a/mdrsclient/models/error.py b/mdrsclient/models/error.py new file mode 100644 index 0000000..02f8863 --- /dev/null +++ b/mdrsclient/models/error.py @@ -0,0 +1,14 @@ +from pydantic.dataclasses import dataclass + + +@dataclass(frozen=True) +class DRFStandardizedError: + code: str + detail: str + attr: str | None + + +@dataclass(frozen=True) +class DRFStandardizedErrors: + type: str + errors: list[DRFStandardizedError] diff --git a/mdrsclient/models/file.py b/mdrsclient/models/file.py new file mode 100644 index 0000000..739feaf --- /dev/null +++ b/mdrsclient/models/file.py @@ -0,0 +1,25 @@ +from pydantic.dataclasses import dataclass + +from mdrsclient.models.utils import iso8601_to_user_friendly + + +@dataclass(frozen=True) +class File: + id: str + name: str + type: str + size: int + thumbnail: str | None + description: str + metadata: dict + download_url: str + created_at: str + updated_at: str + + @property + def created_at_name(self) -> str: + return iso8601_to_user_friendly(self.created_at) + + @property + def updated_at_name(self) -> str: + return iso8601_to_user_friendly(self.updated_at) diff --git a/mdrsclient/models/folder.py b/mdrsclient/models/folder.py new file mode 100644 index 0000000..eebebb9 --- /dev/null +++ b/mdrsclient/models/folder.py @@ -0,0 +1,58 @@ +from typing import Final + +from pydantic.dataclasses import dataclass + +from mdrsclient.models import File +from mdrsclient.models.utils import iso8601_to_user_friendly + +ACCESS_LEVEL_NAMES: Final[dict[int, str]] = { + -1: "Storage", + 0: "Private", + 1: "CBS Open", + 2: "PW Open", + 3: "Public", +} + + +@dataclass(frozen=True) +class FolderSimple: + id: str + pid: str | None + name: str + access_level: int + lock: bool + lab_id: int + description: str + created_at: str + updated_at: str + restrict_opened_at: str | None + + @property + def access_level_name(self) -> str: + return ACCESS_LEVEL_NAMES[self.access_level] + + @property + def lock_name(self) -> str: + return "locked" if self.lock else "unlocked" + + @property + def created_at_name(self) -> str: + return iso8601_to_user_friendly(self.created_at) + + @property + def updated_at_name(self) -> str: + return iso8601_to_user_friendly(self.updated_at) + + +@dataclass(frozen=True) +class Folder(FolderSimple): + metadata: list[dict] + sub_folders: list[FolderSimple] + files: list[File] + path: str + + def find_sub_folder(self, name: str) -> FolderSimple | None: + return next((x for x in self.sub_folders if x.name == name), None) + + def find_file(self, name: str) -> File | None: + return next((x for x in self.files if x.name == name), None) diff --git a/mdrsclient/models/laboratory.py b/mdrsclient/models/laboratory.py new file mode 100644 index 0000000..3419d27 --- /dev/null +++ b/mdrsclient/models/laboratory.py @@ -0,0 +1,34 @@ +from typing import Generator + +from pydantic.dataclasses import dataclass + + +@dataclass(frozen=True) +class Laboratory: + id: int + name: str + pi_name: str + full_name: str + + +@dataclass(frozen=True) +class Laboratories: + items: list[Laboratory] + + def __iter__(self) -> Generator[Laboratory, None, None]: + yield from self.items + + def empty(self) -> bool: + return len(self.items) == 0 + + def clear(self) -> None: + self.items.clear() + + def append(self, item: Laboratory) -> None: + self.items.append(item) + + def find_by_id(self, id: int) -> Laboratory | None: + return next((x for x in self.items if x.id == id), None) + + def find_by_name(self, name: str) -> Laboratory | None: + return next((x for x in self.items if x.name == name), None) diff --git a/mdrsclient/models/user.py b/mdrsclient/models/user.py new file mode 100644 index 0000000..d74db26 --- /dev/null +++ b/mdrsclient/models/user.py @@ -0,0 +1,50 @@ +import time + +import jwt +from pydantic import parse_obj_as +from pydantic.dataclasses import dataclass + + +@dataclass(frozen=True) +class DecodedJWT: + token_type: str + exp: int + iat: int + jti: str + user_id: int + + +@dataclass(frozen=True) +class Token: + access: str + refresh: str + + @property + def user_id(self) -> int: + access_decoded = self.__decode(self.access) + return access_decoded.user_id + + @property + def is_expired(self) -> bool: + now = int(time.time()) + 10 + refresh_decoded = self.__decode(self.refresh) + return now > refresh_decoded.exp + + @property + def is_refresh_required(self) -> bool: + now = int(time.time()) + 10 + access_decoded = self.__decode(self.access) + refresh_decoded = self.__decode(self.refresh) + return now > access_decoded.exp and now < refresh_decoded.exp + + def __decode(self, token: str) -> DecodedJWT: + data = jwt.decode(token, options={"verify_signature": False}) + return parse_obj_as(DecodedJWT, data) + + +@dataclass(frozen=True) +class User: + id: int + username: str + laboratory_id: int + laboratory: str diff --git a/mdrsclient/models/utils.py b/mdrsclient/models/utils.py new file mode 100644 index 0000000..ee282ab --- /dev/null +++ b/mdrsclient/models/utils.py @@ -0,0 +1,5 @@ +import datetime + + +def iso8601_to_user_friendly(text: str) -> str: + return datetime.datetime.fromisoformat(text).strftime("%Y/%m/%d %H:%M:%S") diff --git a/mdrsclient/session.py b/mdrsclient/session.py new file mode 100644 index 0000000..ca36d05 --- /dev/null +++ b/mdrsclient/session.py @@ -0,0 +1,58 @@ +import requests + +from mdrsclient.cache import CacheFile +from mdrsclient.exceptions import MissingConfigurationException +from mdrsclient.models import Laboratories, Token, User + + +class MDRSSession(requests.Session): + url: str + __cache: CacheFile + + def __init__(self, remote: str, url: str) -> None: + super().__init__() + self.url = url + self.__cache = CacheFile(remote) + self.__prepare_headers() + + def build_url(self, *args) -> str: + if self.url == "": + raise MissingConfigurationException("remote host is not configured") + parts = [self.url] + parts.extend(args) + return "/".join(parts) + + def logout(self) -> None: + del self.__cache.user + del self.__cache.token + self.headers.update({"Authorization": ""}) + + @property + def user(self) -> User | None: + return self.__cache.user + + @user.setter + def user(self, user: User) -> None: + self.__cache.user = user + + @property + def token(self) -> Token | None: + return self.__cache.token + + @token.setter + def token(self, token: Token) -> None: + self.__cache.token = token + self.__prepare_headers() + + @property + def laboratories(self) -> Laboratories: + return self.__cache.laboratories + + @laboratories.setter + def laboratories(self, laboratories: Laboratories) -> None: + self.__cache.laboratories = laboratories + + def __prepare_headers(self) -> None: + self.headers.update({"accept": "application/json"}) + if self.token is not None: + self.headers.update({"Authorization": f"Bearer {self.token.access}"}) diff --git a/mdrsclient/settings.py b/mdrsclient/settings.py new file mode 100644 index 0000000..37ad652 --- /dev/null +++ b/mdrsclient/settings.py @@ -0,0 +1,20 @@ +import os + +from pydantic import BaseSettings + + +class Settings(BaseSettings): + config_dir_path: str = "~/.mdrs-client" + number_of_process: int = 10 + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + + +settings = Settings() + +NUMBER_OF_PROCESS = settings.number_of_process + +CONFIG_DIR_PATH = os.path.expanduser(settings.config_dir_path) +CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, "config.ini") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0db30fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +requests +python-dotenv +pydantic +PyJWT +validators diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..103ca96 --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +import os +from typing import Final + +from setuptools import find_packages, setup + +BASE_DIR: Final[str] = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(BASE_DIR, "VERSION")) as f: + __version__ = f.read().strip() + +with open(os.path.join(BASE_DIR, "requirements.txt")) as f: + __requirements__ = f.read().splitlines() + +with open(os.path.join(BASE_DIR, "README.md")) as f: + __readme__ = f.read() + +setup( + name="mdrsclient", + version=__version__, + description="A MDRS command-line tool", + long_description=__readme__, + author="Neuroinformatics Unit, RIKEN CBS", + license="MIT", + classifiers=[ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "OSI Approved :: MIT License", + "Topic :: Utilities", + ], + packages=find_packages(), + include_package_data=True, + package_data={ + "": [ + os.path.join(BASE_DIR, "README.md"), + os.path.join(BASE_DIR, "LICENSE"), + os.path.join(BASE_DIR, "VERSION"), + os.path.join(BASE_DIR, "requirements.txt"), + ] + }, + install_requires=__requirements__, + entry_points={"console_scripts": ["mdrs=mdrsclient.__main__:main"]}, +)