import os import tempfile import unittest from io import StringIO from unittest.mock import MagicMock, patch from mdrsclient.api import FilesApi from mdrsclient.exceptions import MDRSException, UnexpectedException from mdrsclient.models import File, Folder, FolderSimple, Laboratory from mdrsclient.transfer import Downloader, Uploader TIMESTAMP = "2026-01-01T00:00:00+09:00" def make_file(id: str, name: str, size: int = 1) -> File: return File( id=id, name=name, type="text/plain", size=size, thumbnail=None, description="", metadata={}, download_url=f"v3/files/{id}/download/", created_at=TIMESTAMP, updated_at=TIMESTAMP, ) def make_folder_simple(id: str, name: str) -> FolderSimple: return FolderSimple( id=id, pid=None, name=name, access_level=1, lock=False, size=0, laboratory_id=1, description="", created_at=TIMESTAMP, updated_at=TIMESTAMP, restrict_opened_at=None, ) def make_folder(id: str, name: str, path: str, sub_folders: list[FolderSimple] | None = None) -> Folder: return Folder( id=id, pid=None, name=name, access_level=1, lock=False, size=0, laboratory_id=1, description="", created_at=TIMESTAMP, updated_at=TIMESTAMP, restrict_opened_at=None, metadata=[], sub_folders=sub_folders if sub_folders is not None else [], path=path, ) LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory") class TestUploadReportsFailure(unittest.TestCase): """A file that never reached the server must not leave the command reporting success.""" def make_client(self, folder: Folder, files: list[File]) -> MagicMock: client = MagicMock() client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/") client.find_laboratory.return_value = LABORATORY client.find_folder.return_value = folder client.find_files.return_value = files return client def test_a_refused_upload_is_raised_to_the_caller(self): client = self.make_client(make_folder("f1", "root", "/"), []) with tempfile.TemporaryDirectory() as tmp: local = os.path.join(tmp, "data.txt") with open(local, "w") as f: f.write("x") with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.create.side_effect = MDRSException("Access is denied.") with self.assertRaises(UnexpectedException): Uploader(client).upload(local, "myremote:/mylab/", False, False) def test_an_error_the_api_layer_did_not_raise_is_still_counted(self): """A file that vanished between the walk and the upload must not end the run.""" client = self.make_client(make_folder("f1", "root", "/"), []) with tempfile.TemporaryDirectory() as tmp: local = os.path.join(tmp, "data.txt") with open(local, "w") as f: f.write("x") with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.create.side_effect = FileNotFoundError(local) with self.assertRaises(UnexpectedException): Uploader(client).upload(local, "myremote:/mylab/", False, False) def test_a_failed_upload_names_the_file(self): client = self.make_client(make_folder("f1", "root", "/"), []) with tempfile.TemporaryDirectory() as tmp: local = os.path.join(tmp, "data.txt") with open(local, "w") as f: f.write("x") with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.create.side_effect = MDRSException("Access is denied.") with patch("sys.stdout", new=StringIO()) as fake_out: with self.assertRaises(UnexpectedException): Uploader(client).upload(local, "myremote:/mylab/", False, False) self.assertIn(local, fake_out.getvalue()) def test_an_upload_that_worked_stays_quiet(self): client = self.make_client(make_folder("f1", "root", "/"), []) with tempfile.TemporaryDirectory() as tmp: local = os.path.join(tmp, "data.txt") with open(local, "w") as f: f.write("x") with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.create.return_value = "new-id" Uploader(client).upload(local, "myremote:/mylab/", False, False) files_api_class.return_value.create.assert_called_once() class TestDownloadReportsFailure(unittest.TestCase): def make_client(self, remote_path_parts, folder: Folder, files_by_folder: dict[str, list[File]]) -> MagicMock: client = MagicMock() client.is_doi.return_value = False client.parse_remote_host_with_path.return_value = remote_path_parts client.find_laboratory.return_value = LABORATORY client.find_folder.return_value = folder client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, []) return client def test_a_single_file_that_failed_is_raised_to_the_caller(self): parent = make_folder("p1", "root", "/") client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]}) with tempfile.TemporaryDirectory() as tmp: with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.download.side_effect = OSError("Permission denied.") with self.assertRaises(UnexpectedException): Downloader(client).download("myremote:/mylab/data.txt", tmp) def test_a_single_file_that_arrived_stays_quiet(self): parent = make_folder("p1", "root", "/") client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]}) with tempfile.TemporaryDirectory() as tmp: with patch("mdrsclient.transfer.FilesApi") as files_api_class: files_api_class.return_value.download.return_value = True Downloader(client).download("myremote:/mylab/data.txt", tmp) files_api_class.return_value.download.assert_called_once() def test_a_failed_file_does_not_abandon_the_remaining_sub_folders(self): parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")]) folders = { "f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]), "fa": make_folder("fa", "a", "/root/a/"), "fb": make_folder("fb", "b", "/root/b/"), } files_by_folder = { "p1": [], "f1": [make_file("bad", "bad.txt")], "fa": [make_file("good1", "good1.txt")], "fb": [make_file("good2", "good2.txt")], } client = self.make_client(("myremote", "mylab", "/root"), parent, files_by_folder) attempted: list[str] = [] def download(file: File, path: str) -> bool: attempted.append(file.name) if file.name == "bad.txt": raise OSError("Permission denied.") return True with tempfile.TemporaryDirectory() as tmp: with ( patch("mdrsclient.transfer.FoldersApi") as folders_api_class, patch("mdrsclient.transfer.FilesApi") as files_api_class, ): folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id] files_api_class.return_value.download.side_effect = download with self.assertRaises(UnexpectedException): Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True) self.assertEqual(sorted(attempted), ["bad.txt", "good1.txt", "good2.txt"]) class TestRecursiveDownloadResilience(unittest.TestCase): """A folder the client cannot prepare locally must not abandon its siblings.""" def test_a_folder_that_cannot_be_created_does_not_stop_the_walk(self): parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")]) folders = { "f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]), "fa": make_folder("fa", "a", "/root/a/"), "fb": make_folder("fb", "b", "/root/b/"), } files_by_folder = {"p1": [], "f1": [], "fa": [make_file("good1", "good1.txt")], "fb": []} client = MagicMock() client.is_doi.return_value = False client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/root") client.find_laboratory.return_value = LABORATORY client.find_folder.return_value = parent client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, []) real_makedirs = os.makedirs def makedirs(path, *args, **kwargs): if os.path.basename(path) == "b": raise PermissionError("Permission denied") return real_makedirs(path, *args, **kwargs) attempted: list[str] = [] with tempfile.TemporaryDirectory() as tmp: with ( patch("mdrsclient.transfer.FoldersApi") as folders_api_class, patch("mdrsclient.transfer.FilesApi") as files_api_class, patch("mdrsclient.transfer.os.makedirs", side_effect=makedirs), ): folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id] files_api_class.return_value.download.side_effect = lambda file, path: attempted.append(file.name) with self.assertRaises(UnexpectedException): Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True) self.assertEqual(attempted, ["good1.txt"]) class TestFileDownloadPermission(unittest.TestCase): """A file the client could not write is a failure, not a line of successful output.""" def make_connection(self, chunks) -> MagicMock: connection = MagicMock() connection.token = None response = MagicMock() response.status_code = 200 response.iter_content.return_value = chunks connection.get.return_value = response return connection def test_a_download_that_cannot_be_written_is_reported(self): connection = self.make_connection([b"payload"]) with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "precious.dat") with open(path, "wb") as f: f.write(b"do not touch") os.chmod(path, 0o444) try: with open(path, "r+b"): self.skipTest("running with rights that ignore the file mode") except OSError: pass with self.assertRaises(UnexpectedException): FilesApi(connection).download(make_file("x1", "precious.dat"), path) os.chmod(path, 0o644) with open(path, "rb") as f: self.assertEqual(f.read(), b"do not touch") connection.get.assert_not_called() def test_an_interrupted_download_leaves_the_existing_file_intact(self): def chunks(): yield b"half a file" raise ConnectionError("connection reset") connection = self.make_connection(chunks()) with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "existing.dat") with open(path, "wb") as f: f.write(b"the copy already here") with self.assertRaises(ConnectionError): FilesApi(connection).download(make_file("x1", "existing.dat"), path) with open(path, "rb") as f: self.assertEqual(f.read(), b"the copy already here") self.assertEqual(os.listdir(tmp), ["existing.dat"]) def test_a_finished_download_replaces_the_destination(self): connection = self.make_connection([b"new ", b"contents"]) with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "existing.dat") with open(path, "wb") as f: f.write(b"old") FilesApi(connection).download(make_file("x1", "existing.dat"), path) with open(path, "rb") as f: self.assertEqual(f.read(), b"new contents") self.assertEqual(os.listdir(tmp), ["existing.dat"]) if __name__ == "__main__": unittest.main()