orrisroot
914dd729aa
Upload and download failures were printed and then forgotten: a batch that lost files still exited 0, and a file the client could not write was listed as if it had arrived. Nothing downstream could tell. - Return a verdict from every transfer worker and raise once at the end, so a run that lost a file exits 2. - Raise the error when a downloaded file cannot be written locally, instead of printing it and reporting the path as a success. - Write a download beside its destination and move it into place once complete, and refuse a destination that cannot be written, so a failed transfer no longer leaves a truncated file behind. - Carry on through the remaining sub-folders when a recursive download loses a file or cannot create a folder locally. - Name the file and give the reason in every failure message. - Count a failure the API layer did not raise, such as a file that disappeared between the directory walk and its turn to be sent. - Extract the directory walk from `Uploader.upload` to keep it within the complexity limit. - Cover all of the above in `tests/test_transfer.py`.
298 lines
13 KiB
Python
298 lines
13 KiB
Python
import os
|
|
import tempfile
|
|
import unittest
|
|
from io import StringIO
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from mdrsclient.api import FilesApi
|
|
from mdrsclient.exceptions import MDRSException, UnexpectedException
|
|
from mdrsclient.models import File, Folder, FolderSimple, Laboratory
|
|
from mdrsclient.transfer import Downloader, Uploader
|
|
|
|
TIMESTAMP = "2026-01-01T00:00:00+09:00"
|
|
|
|
|
|
def make_file(id: str, name: str, size: int = 1) -> File:
|
|
return File(
|
|
id=id,
|
|
name=name,
|
|
type="text/plain",
|
|
size=size,
|
|
thumbnail=None,
|
|
description="",
|
|
metadata={},
|
|
download_url=f"v3/files/{id}/download/",
|
|
created_at=TIMESTAMP,
|
|
updated_at=TIMESTAMP,
|
|
)
|
|
|
|
|
|
def make_folder_simple(id: str, name: str) -> FolderSimple:
|
|
return FolderSimple(
|
|
id=id,
|
|
pid=None,
|
|
name=name,
|
|
access_level=1,
|
|
lock=False,
|
|
size=0,
|
|
laboratory_id=1,
|
|
description="",
|
|
created_at=TIMESTAMP,
|
|
updated_at=TIMESTAMP,
|
|
restrict_opened_at=None,
|
|
)
|
|
|
|
|
|
def make_folder(id: str, name: str, path: str, sub_folders: list[FolderSimple] | None = None) -> Folder:
|
|
return Folder(
|
|
id=id,
|
|
pid=None,
|
|
name=name,
|
|
access_level=1,
|
|
lock=False,
|
|
size=0,
|
|
laboratory_id=1,
|
|
description="",
|
|
created_at=TIMESTAMP,
|
|
updated_at=TIMESTAMP,
|
|
restrict_opened_at=None,
|
|
metadata=[],
|
|
sub_folders=sub_folders if sub_folders is not None else [],
|
|
path=path,
|
|
)
|
|
|
|
|
|
LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory")
|
|
|
|
|
|
class TestUploadReportsFailure(unittest.TestCase):
|
|
"""A file that never reached the server must not leave the command reporting success."""
|
|
|
|
def make_client(self, folder: Folder, files: list[File]) -> MagicMock:
|
|
client = MagicMock()
|
|
client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/")
|
|
client.find_laboratory.return_value = LABORATORY
|
|
client.find_folder.return_value = folder
|
|
client.find_files.return_value = files
|
|
return client
|
|
|
|
def test_a_refused_upload_is_raised_to_the_caller(self):
|
|
client = self.make_client(make_folder("f1", "root", "/"), [])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
local = os.path.join(tmp, "data.txt")
|
|
with open(local, "w") as f:
|
|
f.write("x")
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
|
|
with self.assertRaises(UnexpectedException):
|
|
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
|
|
|
def test_an_error_the_api_layer_did_not_raise_is_still_counted(self):
|
|
"""A file that vanished between the walk and the upload must not end the run."""
|
|
client = self.make_client(make_folder("f1", "root", "/"), [])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
local = os.path.join(tmp, "data.txt")
|
|
with open(local, "w") as f:
|
|
f.write("x")
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.create.side_effect = FileNotFoundError(local)
|
|
with self.assertRaises(UnexpectedException):
|
|
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
|
|
|
def test_a_failed_upload_names_the_file(self):
|
|
client = self.make_client(make_folder("f1", "root", "/"), [])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
local = os.path.join(tmp, "data.txt")
|
|
with open(local, "w") as f:
|
|
f.write("x")
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
|
|
with patch("sys.stdout", new=StringIO()) as fake_out:
|
|
with self.assertRaises(UnexpectedException):
|
|
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
|
self.assertIn(local, fake_out.getvalue())
|
|
|
|
def test_an_upload_that_worked_stays_quiet(self):
|
|
client = self.make_client(make_folder("f1", "root", "/"), [])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
local = os.path.join(tmp, "data.txt")
|
|
with open(local, "w") as f:
|
|
f.write("x")
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.create.return_value = "new-id"
|
|
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
|
files_api_class.return_value.create.assert_called_once()
|
|
|
|
|
|
class TestDownloadReportsFailure(unittest.TestCase):
|
|
def make_client(self, remote_path_parts, folder: Folder, files_by_folder: dict[str, list[File]]) -> MagicMock:
|
|
client = MagicMock()
|
|
client.is_doi.return_value = False
|
|
client.parse_remote_host_with_path.return_value = remote_path_parts
|
|
client.find_laboratory.return_value = LABORATORY
|
|
client.find_folder.return_value = folder
|
|
client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
|
|
return client
|
|
|
|
def test_a_single_file_that_failed_is_raised_to_the_caller(self):
|
|
parent = make_folder("p1", "root", "/")
|
|
client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.download.side_effect = OSError("Permission denied.")
|
|
with self.assertRaises(UnexpectedException):
|
|
Downloader(client).download("myremote:/mylab/data.txt", tmp)
|
|
|
|
def test_a_single_file_that_arrived_stays_quiet(self):
|
|
parent = make_folder("p1", "root", "/")
|
|
client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
|
files_api_class.return_value.download.return_value = True
|
|
Downloader(client).download("myremote:/mylab/data.txt", tmp)
|
|
files_api_class.return_value.download.assert_called_once()
|
|
|
|
def test_a_failed_file_does_not_abandon_the_remaining_sub_folders(self):
|
|
parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
|
|
folders = {
|
|
"f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
|
|
"fa": make_folder("fa", "a", "/root/a/"),
|
|
"fb": make_folder("fb", "b", "/root/b/"),
|
|
}
|
|
files_by_folder = {
|
|
"p1": [],
|
|
"f1": [make_file("bad", "bad.txt")],
|
|
"fa": [make_file("good1", "good1.txt")],
|
|
"fb": [make_file("good2", "good2.txt")],
|
|
}
|
|
client = self.make_client(("myremote", "mylab", "/root"), parent, files_by_folder)
|
|
attempted: list[str] = []
|
|
|
|
def download(file: File, path: str) -> bool:
|
|
attempted.append(file.name)
|
|
if file.name == "bad.txt":
|
|
raise OSError("Permission denied.")
|
|
return True
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with (
|
|
patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
|
|
patch("mdrsclient.transfer.FilesApi") as files_api_class,
|
|
):
|
|
folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
|
|
files_api_class.return_value.download.side_effect = download
|
|
with self.assertRaises(UnexpectedException):
|
|
Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
|
|
|
|
self.assertEqual(sorted(attempted), ["bad.txt", "good1.txt", "good2.txt"])
|
|
|
|
|
|
class TestRecursiveDownloadResilience(unittest.TestCase):
|
|
"""A folder the client cannot prepare locally must not abandon its siblings."""
|
|
|
|
def test_a_folder_that_cannot_be_created_does_not_stop_the_walk(self):
|
|
parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
|
|
folders = {
|
|
"f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
|
|
"fa": make_folder("fa", "a", "/root/a/"),
|
|
"fb": make_folder("fb", "b", "/root/b/"),
|
|
}
|
|
files_by_folder = {"p1": [], "f1": [], "fa": [make_file("good1", "good1.txt")], "fb": []}
|
|
client = MagicMock()
|
|
client.is_doi.return_value = False
|
|
client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/root")
|
|
client.find_laboratory.return_value = LABORATORY
|
|
client.find_folder.return_value = parent
|
|
client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
|
|
|
|
real_makedirs = os.makedirs
|
|
|
|
def makedirs(path, *args, **kwargs):
|
|
if os.path.basename(path) == "b":
|
|
raise PermissionError("Permission denied")
|
|
return real_makedirs(path, *args, **kwargs)
|
|
|
|
attempted: list[str] = []
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with (
|
|
patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
|
|
patch("mdrsclient.transfer.FilesApi") as files_api_class,
|
|
patch("mdrsclient.transfer.os.makedirs", side_effect=makedirs),
|
|
):
|
|
folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
|
|
files_api_class.return_value.download.side_effect = lambda file, path: attempted.append(file.name)
|
|
with self.assertRaises(UnexpectedException):
|
|
Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
|
|
|
|
self.assertEqual(attempted, ["good1.txt"])
|
|
|
|
|
|
class TestFileDownloadPermission(unittest.TestCase):
|
|
"""A file the client could not write is a failure, not a line of successful output."""
|
|
|
|
def make_connection(self, chunks) -> MagicMock:
|
|
connection = MagicMock()
|
|
connection.token = None
|
|
response = MagicMock()
|
|
response.status_code = 200
|
|
response.iter_content.return_value = chunks
|
|
connection.get.return_value = response
|
|
return connection
|
|
|
|
def test_a_download_that_cannot_be_written_is_reported(self):
|
|
connection = self.make_connection([b"payload"])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "precious.dat")
|
|
with open(path, "wb") as f:
|
|
f.write(b"do not touch")
|
|
os.chmod(path, 0o444)
|
|
try:
|
|
with open(path, "r+b"):
|
|
self.skipTest("running with rights that ignore the file mode")
|
|
except OSError:
|
|
pass
|
|
|
|
with self.assertRaises(UnexpectedException):
|
|
FilesApi(connection).download(make_file("x1", "precious.dat"), path)
|
|
|
|
os.chmod(path, 0o644)
|
|
with open(path, "rb") as f:
|
|
self.assertEqual(f.read(), b"do not touch")
|
|
connection.get.assert_not_called()
|
|
|
|
def test_an_interrupted_download_leaves_the_existing_file_intact(self):
|
|
def chunks():
|
|
yield b"half a file"
|
|
raise ConnectionError("connection reset")
|
|
|
|
connection = self.make_connection(chunks())
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "existing.dat")
|
|
with open(path, "wb") as f:
|
|
f.write(b"the copy already here")
|
|
|
|
with self.assertRaises(ConnectionError):
|
|
FilesApi(connection).download(make_file("x1", "existing.dat"), path)
|
|
|
|
with open(path, "rb") as f:
|
|
self.assertEqual(f.read(), b"the copy already here")
|
|
self.assertEqual(os.listdir(tmp), ["existing.dat"])
|
|
|
|
def test_a_finished_download_replaces_the_destination(self):
|
|
connection = self.make_connection([b"new ", b"contents"])
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "existing.dat")
|
|
with open(path, "wb") as f:
|
|
f.write(b"old")
|
|
|
|
FilesApi(connection).download(make_file("x1", "existing.dat"), path)
|
|
|
|
with open(path, "rb") as f:
|
|
self.assertEqual(f.read(), b"new contents")
|
|
self.assertEqual(os.listdir(tmp), ["existing.dat"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|