import os from argparse import Namespace, _SubParsersAction from concurrent.futures import ThreadPoolExecutor from pydantic.dataclasses import dataclass from mdrsclient.api import FileApi, FolderApi from mdrsclient.commands.base import BaseCommand from mdrsclient.connection import MDRSConnection from mdrsclient.exceptions import IllegalArgumentException from mdrsclient.models import File from mdrsclient.settings import CONCURRENT @dataclass(frozen=True) class DownloadFileInfo: file: File path: str class DownloadCommand(BaseCommand): @classmethod def register(cls, parsers: _SubParsersAction) -> None: command = cls() download_parser = parsers.add_parser("download", help="download the file or folder") download_parser.add_argument( "-r", "--recursive", help="download folders and their contents recursive", action="store_true" ) download_parser.add_argument("-p", "--password", help="password to use when open locked folder") download_parser.add_argument("remote_path", help="remote file path (remote:/lab/path/file)") download_parser.add_argument("local_path", help="local folder path (/foo/bar/)") download_parser.set_defaults(func=command.download) def download(self, args: Namespace) -> None: (remote, laboratory_name, r_path) = self._parse_remote_host_with_path(args.remote_path) r_path = r_path.rstrip("/") r_dirname = os.path.dirname(r_path) r_basename = os.path.basename(r_path) connection = self._create_connection(remote) l_dirname = os.path.realpath(args.local_path) if not os.path.isdir(l_dirname): raise IllegalArgumentException(f"Local directory `{args.local_path}` not found.") laboratory = self._find_laboratory(connection, laboratory_name) r_parent_folder = self._find_folder(connection, laboratory, r_dirname, args.password) file = r_parent_folder.find_file(r_basename) download_files: list[DownloadFileInfo] = [] if file is not None: l_path = os.path.join(l_dirname, r_basename) download_files.append(DownloadFileInfo(file, l_path)) else: folder = r_parent_folder.find_sub_folder(r_basename) if folder is None: raise IllegalArgumentException(f"File or folder `{r_path}` not found.") if not args.recursive: raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.") folder_api = FolderApi(connection) self.__multiple_download_pickup_recursive_files(folder_api, download_files, folder.id, l_dirname) self.__multiple_download(connection, download_files) def __multiple_download_pickup_recursive_files( self, folder_api: FolderApi, infolist: list[DownloadFileInfo], folder_id: str, basedir: str ) -> None: folder = folder_api.retrieve(folder_id) dirname = os.path.join(basedir, folder.name) if not os.path.exists(dirname): os.makedirs(dirname) print(dirname) for file in folder.files: path = os.path.join(dirname, file.name) infolist.append(DownloadFileInfo(file, path)) for sub_folder in folder.sub_folders: self.__multiple_download_pickup_recursive_files(folder_api, infolist, sub_folder.id, dirname) def __multiple_download(self, connection: MDRSConnection, infolist: list[DownloadFileInfo]) -> None: file_api = FileApi(connection) with ThreadPoolExecutor(max_workers=CONCURRENT) as pool: pool.map(lambda x: self.__multiple_download_worker(file_api, x), infolist) def __multiple_download_worker(self, file_api: FileApi, info: DownloadFileInfo) -> None: file_api.download(info.file, info.path) print(info.path)