fix: report failed transfers instead of ending in success

Upload and download failures were printed and then forgotten: a batch
that lost files still exited 0, and a file the client could not write
was listed as if it had arrived. Nothing downstream could tell.

- Return a verdict from every transfer worker and raise once at the
  end, so a run that lost a file exits 2.
- Raise the error when a downloaded file cannot be written locally,
  instead of printing it and reporting the path as a success.
- Write a download beside its destination and move it into place once
  complete, and refuse a destination that cannot be written, so a
  failed transfer no longer leaves a truncated file behind.
- Carry on through the remaining sub-folders when a recursive download
  loses a file or cannot create a folder locally.
- Name the file and give the reason in every failure message.
- Count a failure the API layer did not raise, such as a file that
  disappeared between the directory walk and its turn to be sent.
- Extract the directory walk from `Uploader.upload` to keep it within
  the complexity limit.
- Cover all of the above in `tests/test_transfer.py`.

mdrsclient/api/files.py

@@ -1,5 +1,6 @@
 import mimetypes
 import os
+import threading
 from typing import Any, Final
 from unicodedata import normalize
 
@@ -146,18 +147,53 @@ class FilesApi(BaseApi):
         # print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
         url = file.download_url
         token_check(self.connection)
+        # Refused before anything is fetched. The finished file is moved into place, and a
+        # rename would replace a destination whose mode says it is protected.
+        if os.path.exists(path):
+            try:
+                with open(path, "r+b"):
+                    pass
+            except OSError as e:
+                raise UnexpectedException(f"Cannot write `{path}`: {e}")
         response = self.connection.get(url, stream=True)
         self._raise_response_error(response)
+        # Written beside the destination and moved in once the whole body has arrived, so
+        # a transfer that fails part way leaves whatever was already there untouched and
+        # never leaves a truncated file under the real name.
+        fd, tmp_path = self._open_partial(path)
         try:
-            with open(path, "wb") as f:
+            with os.fdopen(fd, "wb") as f:
                 for chunk in response.iter_content(chunk_size=4096):
                     if chunk:
                         f.write(chunk)
-                        f.flush()
-        except PermissionError:
-            print(f"Cannot create file `{path}`: Permission denied.")
+            os.replace(tmp_path, path)
+        except BaseException:
+            # Only the scratch file goes: anything at the destination was not written here.
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+            raise
         return True
 
+    @staticmethod
+    def _open_partial(path: str) -> tuple[int, str]:
+        """
+        Create a scratch file beside `path` and return it open for writing.
+
+        Beside it, so moving the finished download into place is a rename within one
+        directory. `0o666` rather than a private mode because the umask is what decided
+        the permissions of a downloaded file before, and still should.
+        """
+        base = f"{path}.{os.getpid()}-{threading.get_ident()}"
+        for attempt in range(100):
+            tmp_path = f"{base}-{attempt}.mdrspart"
+            try:
+                return os.open(tmp_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666), tmp_path
+            except FileExistsError:
+                continue
+            except OSError as e:
+                raise UnexpectedException(f"Cannot write `{path}`: {e}")
+        raise UnexpectedException(f"Could not create a temporary file beside `{path}`.")
+
     def _get_mime_type(self, path: str) -> str:
         mt = mimetypes.guess_type(path)
         if mt: