121 lines
5.6 KiB
Python
121 lines
5.6 KiB
Python
import os
|
|
from argparse import Namespace
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any
|
|
|
|
from pydantic.dataclasses import dataclass
|
|
|
|
from mdrsclient.api import FilesApi, FoldersApi
|
|
from mdrsclient.commands.base import BaseCommand
|
|
from mdrsclient.connection import MDRSConnection
|
|
from mdrsclient.exceptions import IllegalArgumentException, MDRSException
|
|
from mdrsclient.models import File, Folder
|
|
from mdrsclient.models.file import find_file
|
|
from mdrsclient.settings import CONCURRENT
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class UploadFileInfo:
|
|
folder: Folder
|
|
files: list[File]
|
|
path: str
|
|
|
|
|
|
class UploadCommand(BaseCommand):
|
|
@classmethod
|
|
def register(cls, parsers: Any) -> None:
|
|
upload_parser = parsers.add_parser("upload", help="upload the file or directory")
|
|
upload_parser.add_argument(
|
|
"-r", "--recursive", help="upload directories and their contents recursive", action="store_true"
|
|
)
|
|
upload_parser.add_argument(
|
|
"-s",
|
|
"--skip-if-exists",
|
|
help="skip the upload if file is already uploaded and file size is the same",
|
|
action="store_true",
|
|
)
|
|
upload_parser.add_argument("local_path", help="local file path (/foo/bar/data.txt)")
|
|
upload_parser.add_argument("remote_path", help="remote folder path (remote:/lab/path/)")
|
|
upload_parser.set_defaults(func=cls.func)
|
|
|
|
@classmethod
|
|
def func(cls, args: Namespace) -> None:
|
|
local_path = str(args.local_path)
|
|
remote_path = str(args.remote_path)
|
|
is_recursive = bool(args.recursive)
|
|
is_skip_if_exists = bool(args.skip_if_exists)
|
|
cls.upload(local_path, remote_path, is_recursive, is_skip_if_exists)
|
|
|
|
@classmethod
|
|
def upload(cls, local_path: str, remote_path: str, is_recursive: bool, is_skip_if_exists: bool) -> None:
|
|
(remote, laboratory_name, r_path) = cls._parse_remote_host_with_path(remote_path)
|
|
l_path = os.path.abspath(local_path)
|
|
if not os.path.exists(l_path):
|
|
raise IllegalArgumentException(f"File or directory `{local_path}` not found.")
|
|
connection = cls._create_connection(remote)
|
|
laboratory = cls._find_laboratory(connection, laboratory_name)
|
|
folder = cls._find_folder(connection, laboratory, r_path)
|
|
files = cls._find_files(connection, folder.id)
|
|
infos: list[UploadFileInfo] = []
|
|
if os.path.isdir(l_path):
|
|
if not is_recursive:
|
|
raise IllegalArgumentException(f"Cannot upload `{local_path}`: Is a directory.")
|
|
folder_api = FoldersApi(connection)
|
|
folder_map: dict[str, Folder] = {}
|
|
folder_map[r_path] = folder
|
|
files_map: dict[str, list[File]] = {}
|
|
files_map[r_path] = files
|
|
l_basename = os.path.basename(l_path)
|
|
for dirpath, _, filenames in os.walk(l_path, followlinks=True):
|
|
sub = l_basename if dirpath == l_path else os.path.join(l_basename, os.path.relpath(dirpath, l_path))
|
|
d_dirname = os.path.join(r_path, sub)
|
|
d_basename = os.path.basename(d_dirname)
|
|
# prepare destination parent path
|
|
d_parent_dirname = os.path.dirname(d_dirname)
|
|
if folder_map.get(d_parent_dirname) is None:
|
|
parent_folder = cls._find_folder(connection, laboratory, d_parent_dirname)
|
|
folder_map[d_parent_dirname] = parent_folder
|
|
parent_files = cls._find_files(connection, parent_folder.id)
|
|
files_map[d_parent_dirname] = parent_files
|
|
# prepare destination path
|
|
if folder_map.get(d_dirname) is None:
|
|
d_folder = folder_map[d_parent_dirname].find_sub_folder(d_basename)
|
|
if d_folder is None:
|
|
d_folder_id = folder_api.create(d_basename, folder_map[d_parent_dirname].id)
|
|
else:
|
|
d_folder_id = d_folder.id
|
|
print(d_dirname)
|
|
folder_map[d_dirname] = folder_api.retrieve(d_folder_id)
|
|
files_map[d_dirname] = cls._find_files(connection, d_folder_id)
|
|
if d_folder is None:
|
|
folder_map[d_parent_dirname].sub_folders.append(folder_map[d_dirname])
|
|
# register upload file list
|
|
for filename in filenames:
|
|
infos.append(
|
|
UploadFileInfo(folder_map[d_dirname], files_map[d_dirname], os.path.join(dirpath, filename))
|
|
)
|
|
else:
|
|
infos.append(UploadFileInfo(folder, files, l_path))
|
|
cls.__multiple_upload(connection, infos, is_skip_if_exists)
|
|
|
|
@classmethod
|
|
def __multiple_upload(
|
|
cls, connection: MDRSConnection, infos: list[UploadFileInfo], is_skip_if_exists: bool
|
|
) -> None:
|
|
file_api = FilesApi(connection)
|
|
with ThreadPoolExecutor(max_workers=CONCURRENT) as pool:
|
|
pool.map(lambda x: cls.__multiple_upload_worker(file_api, x, is_skip_if_exists), infos)
|
|
|
|
@classmethod
|
|
def __multiple_upload_worker(cls, file_api: FilesApi, info: UploadFileInfo, is_skip_if_exists: bool) -> None:
|
|
basename = os.path.basename(info.path)
|
|
file = find_file(info.files, basename)
|
|
try:
|
|
if file is None:
|
|
file_api.create(info.folder.id, info.path)
|
|
elif not is_skip_if_exists or file.size != os.path.getsize(info.path):
|
|
file_api.update(file, info.path)
|
|
print(os.path.join(info.folder.path, basename))
|
|
except MDRSException as e:
|
|
print(f"Error: {e}")
|