fix: report failed transfers instead of ending in success
Upload and download failures were printed and then forgotten: a batch that lost files still exited 0, and a file the client could not write was listed as if it had arrived. Nothing downstream could tell. - Return a verdict from every transfer worker and raise once at the end, so a run that lost a file exits 2. - Raise the error when a downloaded file cannot be written locally, instead of printing it and reporting the path as a success. - Write a download beside its destination and move it into place once complete, and refuse a destination that cannot be written, so a failed transfer no longer leaves a truncated file behind. - Carry on through the remaining sub-folders when a recursive download loses a file or cannot create a folder locally. - Name the file and give the reason in every failure message. - Count a failure the API layer did not raise, such as a file that disappeared between the directory walk and its turn to be sent. - Extract the directory walk from `Uploader.upload` to keep it within the complexity limit. - Cover all of the above in `tests/test_transfer.py`.
This commit is contained in:
@@ -0,0 +1,297 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from mdrsclient.api import FilesApi
|
||||
from mdrsclient.exceptions import MDRSException, UnexpectedException
|
||||
from mdrsclient.models import File, Folder, FolderSimple, Laboratory
|
||||
from mdrsclient.transfer import Downloader, Uploader
|
||||
|
||||
TIMESTAMP = "2026-01-01T00:00:00+09:00"
|
||||
|
||||
|
||||
def make_file(id: str, name: str, size: int = 1) -> File:
|
||||
return File(
|
||||
id=id,
|
||||
name=name,
|
||||
type="text/plain",
|
||||
size=size,
|
||||
thumbnail=None,
|
||||
description="",
|
||||
metadata={},
|
||||
download_url=f"v3/files/{id}/download/",
|
||||
created_at=TIMESTAMP,
|
||||
updated_at=TIMESTAMP,
|
||||
)
|
||||
|
||||
|
||||
def make_folder_simple(id: str, name: str) -> FolderSimple:
|
||||
return FolderSimple(
|
||||
id=id,
|
||||
pid=None,
|
||||
name=name,
|
||||
access_level=1,
|
||||
lock=False,
|
||||
size=0,
|
||||
laboratory_id=1,
|
||||
description="",
|
||||
created_at=TIMESTAMP,
|
||||
updated_at=TIMESTAMP,
|
||||
restrict_opened_at=None,
|
||||
)
|
||||
|
||||
|
||||
def make_folder(id: str, name: str, path: str, sub_folders: list[FolderSimple] | None = None) -> Folder:
|
||||
return Folder(
|
||||
id=id,
|
||||
pid=None,
|
||||
name=name,
|
||||
access_level=1,
|
||||
lock=False,
|
||||
size=0,
|
||||
laboratory_id=1,
|
||||
description="",
|
||||
created_at=TIMESTAMP,
|
||||
updated_at=TIMESTAMP,
|
||||
restrict_opened_at=None,
|
||||
metadata=[],
|
||||
sub_folders=sub_folders if sub_folders is not None else [],
|
||||
path=path,
|
||||
)
|
||||
|
||||
|
||||
LABORATORY = Laboratory(id=1, name="mylab", pi_name="PI", full_name="My Laboratory")
|
||||
|
||||
|
||||
class TestUploadReportsFailure(unittest.TestCase):
|
||||
"""A file that never reached the server must not leave the command reporting success."""
|
||||
|
||||
def make_client(self, folder: Folder, files: list[File]) -> MagicMock:
|
||||
client = MagicMock()
|
||||
client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/")
|
||||
client.find_laboratory.return_value = LABORATORY
|
||||
client.find_folder.return_value = folder
|
||||
client.find_files.return_value = files
|
||||
return client
|
||||
|
||||
def test_a_refused_upload_is_raised_to_the_caller(self):
|
||||
client = self.make_client(make_folder("f1", "root", "/"), [])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local = os.path.join(tmp, "data.txt")
|
||||
with open(local, "w") as f:
|
||||
f.write("x")
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
||||
|
||||
def test_an_error_the_api_layer_did_not_raise_is_still_counted(self):
|
||||
"""A file that vanished between the walk and the upload must not end the run."""
|
||||
client = self.make_client(make_folder("f1", "root", "/"), [])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local = os.path.join(tmp, "data.txt")
|
||||
with open(local, "w") as f:
|
||||
f.write("x")
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.create.side_effect = FileNotFoundError(local)
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
||||
|
||||
def test_a_failed_upload_names_the_file(self):
|
||||
client = self.make_client(make_folder("f1", "root", "/"), [])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local = os.path.join(tmp, "data.txt")
|
||||
with open(local, "w") as f:
|
||||
f.write("x")
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.create.side_effect = MDRSException("Access is denied.")
|
||||
with patch("sys.stdout", new=StringIO()) as fake_out:
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
||||
self.assertIn(local, fake_out.getvalue())
|
||||
|
||||
def test_an_upload_that_worked_stays_quiet(self):
|
||||
client = self.make_client(make_folder("f1", "root", "/"), [])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local = os.path.join(tmp, "data.txt")
|
||||
with open(local, "w") as f:
|
||||
f.write("x")
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.create.return_value = "new-id"
|
||||
Uploader(client).upload(local, "myremote:/mylab/", False, False)
|
||||
files_api_class.return_value.create.assert_called_once()
|
||||
|
||||
|
||||
class TestDownloadReportsFailure(unittest.TestCase):
|
||||
def make_client(self, remote_path_parts, folder: Folder, files_by_folder: dict[str, list[File]]) -> MagicMock:
|
||||
client = MagicMock()
|
||||
client.is_doi.return_value = False
|
||||
client.parse_remote_host_with_path.return_value = remote_path_parts
|
||||
client.find_laboratory.return_value = LABORATORY
|
||||
client.find_folder.return_value = folder
|
||||
client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
|
||||
return client
|
||||
|
||||
def test_a_single_file_that_failed_is_raised_to_the_caller(self):
|
||||
parent = make_folder("p1", "root", "/")
|
||||
client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.download.side_effect = OSError("Permission denied.")
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Downloader(client).download("myremote:/mylab/data.txt", tmp)
|
||||
|
||||
def test_a_single_file_that_arrived_stays_quiet(self):
|
||||
parent = make_folder("p1", "root", "/")
|
||||
client = self.make_client(("myremote", "mylab", "/data.txt"), parent, {"p1": [make_file("x1", "data.txt")]})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch("mdrsclient.transfer.FilesApi") as files_api_class:
|
||||
files_api_class.return_value.download.return_value = True
|
||||
Downloader(client).download("myremote:/mylab/data.txt", tmp)
|
||||
files_api_class.return_value.download.assert_called_once()
|
||||
|
||||
def test_a_failed_file_does_not_abandon_the_remaining_sub_folders(self):
|
||||
parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
|
||||
folders = {
|
||||
"f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
|
||||
"fa": make_folder("fa", "a", "/root/a/"),
|
||||
"fb": make_folder("fb", "b", "/root/b/"),
|
||||
}
|
||||
files_by_folder = {
|
||||
"p1": [],
|
||||
"f1": [make_file("bad", "bad.txt")],
|
||||
"fa": [make_file("good1", "good1.txt")],
|
||||
"fb": [make_file("good2", "good2.txt")],
|
||||
}
|
||||
client = self.make_client(("myremote", "mylab", "/root"), parent, files_by_folder)
|
||||
attempted: list[str] = []
|
||||
|
||||
def download(file: File, path: str) -> bool:
|
||||
attempted.append(file.name)
|
||||
if file.name == "bad.txt":
|
||||
raise OSError("Permission denied.")
|
||||
return True
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with (
|
||||
patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
|
||||
patch("mdrsclient.transfer.FilesApi") as files_api_class,
|
||||
):
|
||||
folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
|
||||
files_api_class.return_value.download.side_effect = download
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
|
||||
|
||||
self.assertEqual(sorted(attempted), ["bad.txt", "good1.txt", "good2.txt"])
|
||||
|
||||
|
||||
class TestRecursiveDownloadResilience(unittest.TestCase):
|
||||
"""A folder the client cannot prepare locally must not abandon its siblings."""
|
||||
|
||||
def test_a_folder_that_cannot_be_created_does_not_stop_the_walk(self):
|
||||
parent = make_folder("p1", "lab", "/", [make_folder_simple("f1", "root")])
|
||||
folders = {
|
||||
"f1": make_folder("f1", "root", "/root/", [make_folder_simple("fa", "a"), make_folder_simple("fb", "b")]),
|
||||
"fa": make_folder("fa", "a", "/root/a/"),
|
||||
"fb": make_folder("fb", "b", "/root/b/"),
|
||||
}
|
||||
files_by_folder = {"p1": [], "f1": [], "fa": [make_file("good1", "good1.txt")], "fb": []}
|
||||
client = MagicMock()
|
||||
client.is_doi.return_value = False
|
||||
client.parse_remote_host_with_path.return_value = ("myremote", "mylab", "/root")
|
||||
client.find_laboratory.return_value = LABORATORY
|
||||
client.find_folder.return_value = parent
|
||||
client.find_files.side_effect = lambda folder_id: files_by_folder.get(folder_id, [])
|
||||
|
||||
real_makedirs = os.makedirs
|
||||
|
||||
def makedirs(path, *args, **kwargs):
|
||||
if os.path.basename(path) == "b":
|
||||
raise PermissionError("Permission denied")
|
||||
return real_makedirs(path, *args, **kwargs)
|
||||
|
||||
attempted: list[str] = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with (
|
||||
patch("mdrsclient.transfer.FoldersApi") as folders_api_class,
|
||||
patch("mdrsclient.transfer.FilesApi") as files_api_class,
|
||||
patch("mdrsclient.transfer.os.makedirs", side_effect=makedirs),
|
||||
):
|
||||
folders_api_class.return_value.retrieve.side_effect = lambda folder_id: folders[folder_id]
|
||||
files_api_class.return_value.download.side_effect = lambda file, path: attempted.append(file.name)
|
||||
with self.assertRaises(UnexpectedException):
|
||||
Downloader(client).download("myremote:/mylab/root", tmp, is_recursive=True)
|
||||
|
||||
self.assertEqual(attempted, ["good1.txt"])
|
||||
|
||||
|
||||
class TestFileDownloadPermission(unittest.TestCase):
|
||||
"""A file the client could not write is a failure, not a line of successful output."""
|
||||
|
||||
def make_connection(self, chunks) -> MagicMock:
|
||||
connection = MagicMock()
|
||||
connection.token = None
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.iter_content.return_value = chunks
|
||||
connection.get.return_value = response
|
||||
return connection
|
||||
|
||||
def test_a_download_that_cannot_be_written_is_reported(self):
|
||||
connection = self.make_connection([b"payload"])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "precious.dat")
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"do not touch")
|
||||
os.chmod(path, 0o444)
|
||||
try:
|
||||
with open(path, "r+b"):
|
||||
self.skipTest("running with rights that ignore the file mode")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with self.assertRaises(UnexpectedException):
|
||||
FilesApi(connection).download(make_file("x1", "precious.dat"), path)
|
||||
|
||||
os.chmod(path, 0o644)
|
||||
with open(path, "rb") as f:
|
||||
self.assertEqual(f.read(), b"do not touch")
|
||||
connection.get.assert_not_called()
|
||||
|
||||
def test_an_interrupted_download_leaves_the_existing_file_intact(self):
|
||||
def chunks():
|
||||
yield b"half a file"
|
||||
raise ConnectionError("connection reset")
|
||||
|
||||
connection = self.make_connection(chunks())
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "existing.dat")
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"the copy already here")
|
||||
|
||||
with self.assertRaises(ConnectionError):
|
||||
FilesApi(connection).download(make_file("x1", "existing.dat"), path)
|
||||
|
||||
with open(path, "rb") as f:
|
||||
self.assertEqual(f.read(), b"the copy already here")
|
||||
self.assertEqual(os.listdir(tmp), ["existing.dat"])
|
||||
|
||||
def test_a_finished_download_replaces_the_destination(self):
|
||||
connection = self.make_connection([b"new ", b"contents"])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "existing.dat")
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"old")
|
||||
|
||||
FilesApi(connection).download(make_file("x1", "existing.dat"), path)
|
||||
|
||||
with open(path, "rb") as f:
|
||||
self.assertEqual(f.read(), b"new contents")
|
||||
self.assertEqual(os.listdir(tmp), ["existing.dat"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user