orrisroot
914dd729aa
Upload and download failures were printed and then forgotten: a batch that lost files still exited 0, and a file the client could not write was listed as if it had arrived. Nothing downstream could tell. - Return a verdict from every transfer worker and raise once at the end, so a run that lost a file exits 2. - Raise the error when a downloaded file cannot be written locally, instead of printing it and reporting the path as a success. - Write a download beside its destination and move it into place once complete, and refuse a destination that cannot be written, so a failed transfer no longer leaves a truncated file behind. - Carry on through the remaining sub-folders when a recursive download loses a file or cannot create a folder locally. - Name the file and give the reason in every failure message. - Count a failure the API layer did not raise, such as a file that disappeared between the directory walk and its turn to be sent. - Extract the directory walk from `Uploader.upload` to keep it within the complexity limit. - Cover all of the above in `tests/test_transfer.py`.
202 lines
8.3 KiB
Python
202 lines
8.3 KiB
Python
import mimetypes
|
|
import os
|
|
import threading
|
|
from typing import Any, Final
|
|
from unicodedata import normalize
|
|
|
|
from pydantic import TypeAdapter
|
|
from pydantic.dataclasses import dataclass
|
|
from requests_toolbelt.multipart.encoder import MultipartEncoder
|
|
|
|
from mdrsclient.api.base import BaseApi
|
|
from mdrsclient.api.utils import retry_on_expired_token, token_check
|
|
from mdrsclient.exceptions import MDRSException, UnexpectedException
|
|
from mdrsclient.models import File
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FilesApiCreateResponse:
|
|
id: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FilesApiListResponse:
|
|
count: int
|
|
next: str | None
|
|
previous: str | None
|
|
results: list[File]
|
|
|
|
|
|
class FilesApi(BaseApi):
|
|
ENTRYPOINT: Final[str] = "v3/files/"
|
|
FALLBACK_MIMETYPE: Final[str] = "application/octet-stream"
|
|
|
|
@retry_on_expired_token
|
|
def list(self, folder_id: str, page_num: int) -> FilesApiListResponse:
|
|
url = self.ENTRYPOINT
|
|
token_check(self.connection)
|
|
params: dict[str, str | int] = {"folder_id": folder_id, "page": page_num}
|
|
response = self.connection.get(url, params=params)
|
|
self._raise_response_error(response)
|
|
return TypeAdapter(FilesApiListResponse).validate_python(response.json())
|
|
|
|
@retry_on_expired_token
|
|
def retrieve(self, id: str) -> File:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + id + "/"
|
|
token_check(self.connection)
|
|
response = self.connection.get(url)
|
|
self._raise_response_error(response)
|
|
return TypeAdapter(File).validate_python(response.json())
|
|
|
|
@retry_on_expired_token
|
|
def create(self, folder_id: str, path: str) -> str:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT
|
|
token_check(self.connection)
|
|
data: dict[str, str | int] | MultipartEncoder = {}
|
|
try:
|
|
with open(os.path.realpath(path), mode="rb") as fp:
|
|
data = MultipartEncoder(
|
|
fields={
|
|
"folder_id": folder_id,
|
|
"file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path)),
|
|
}
|
|
)
|
|
response = self.connection.post(url, data=data, headers={"Content-Type": data.content_type})
|
|
self._raise_response_error(response)
|
|
ret = TypeAdapter(FilesApiCreateResponse).validate_python(response.json())
|
|
except OSError:
|
|
raise UnexpectedException(f"Could not open `{path}` file.")
|
|
except MemoryError:
|
|
raise UnexpectedException("Out of memory.")
|
|
except MDRSException:
|
|
# Already says what went wrong, and the caller may want to act on the kind.
|
|
raise
|
|
except Exception as e:
|
|
raise UnexpectedException("Unspecified error.") from e
|
|
return ret.id
|
|
|
|
@retry_on_expired_token
|
|
def update(self, file: File, path: str | None) -> bool:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + file.id + "/"
|
|
token_check(self.connection)
|
|
data: dict[str, str | int] | MultipartEncoder = {}
|
|
if path is not None:
|
|
# update file body
|
|
try:
|
|
with open(os.path.realpath(path), mode="rb") as fp:
|
|
data = MultipartEncoder(
|
|
fields={"file": (normalize("NFC", os.path.basename(path)), fp, self._get_mime_type(path))}
|
|
)
|
|
response = self.connection.put(url, data=data, headers={"Content-Type": data.content_type})
|
|
except OSError:
|
|
raise UnexpectedException(f"Could not open `{path}` file.")
|
|
except MemoryError:
|
|
raise UnexpectedException("Out of memory.")
|
|
except Exception as e:
|
|
raise UnexpectedException("Unspecified error.") from e
|
|
else:
|
|
# update metadata
|
|
data = {"name": file.name, "description": file.description}
|
|
response = self.connection.put(url, data=data)
|
|
self._raise_response_error(response)
|
|
return True
|
|
|
|
@retry_on_expired_token
|
|
def destroy(self, file: File) -> bool:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + file.id + "/"
|
|
token_check(self.connection)
|
|
response = self.connection.delete(url)
|
|
self._raise_response_error(response)
|
|
return True
|
|
|
|
@retry_on_expired_token
|
|
def move(self, file: File, folder_id: str, name: str) -> bool:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + file.id + "/move/"
|
|
data: dict[str, str | int] = {"folder": folder_id, "name": name}
|
|
token_check(self.connection)
|
|
response = self.connection.post(url, data=data)
|
|
self._raise_response_error(response)
|
|
return True
|
|
|
|
@retry_on_expired_token
|
|
def copy(self, file: File, folder_id: str, name: str) -> bool:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + file.id + "/copy/"
|
|
data: dict[str, str | int] = {"folder": folder_id, "name": name}
|
|
token_check(self.connection)
|
|
response = self.connection.post(url, data=data)
|
|
self._raise_response_error(response)
|
|
return True
|
|
|
|
@retry_on_expired_token
|
|
def metadata(self, file: File) -> dict[str, Any]:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = self.ENTRYPOINT + file.id + "/metadata/"
|
|
token_check(self.connection)
|
|
response = self.connection.get(url)
|
|
self._raise_response_error(response)
|
|
return response.json()
|
|
|
|
@retry_on_expired_token
|
|
def download(self, file: File, path: str) -> bool:
|
|
# print(self.__class__.__name__ + "::" + sys._getframe().f_code.co_name)
|
|
url = file.download_url
|
|
token_check(self.connection)
|
|
# Refused before anything is fetched. The finished file is moved into place, and a
|
|
# rename would replace a destination whose mode says it is protected.
|
|
if os.path.exists(path):
|
|
try:
|
|
with open(path, "r+b"):
|
|
pass
|
|
except OSError as e:
|
|
raise UnexpectedException(f"Cannot write `{path}`: {e}")
|
|
response = self.connection.get(url, stream=True)
|
|
self._raise_response_error(response)
|
|
# Written beside the destination and moved in once the whole body has arrived, so
|
|
# a transfer that fails part way leaves whatever was already there untouched and
|
|
# never leaves a truncated file under the real name.
|
|
fd, tmp_path = self._open_partial(path)
|
|
try:
|
|
with os.fdopen(fd, "wb") as f:
|
|
for chunk in response.iter_content(chunk_size=4096):
|
|
if chunk:
|
|
f.write(chunk)
|
|
os.replace(tmp_path, path)
|
|
except BaseException:
|
|
# Only the scratch file goes: anything at the destination was not written here.
|
|
if os.path.exists(tmp_path):
|
|
os.unlink(tmp_path)
|
|
raise
|
|
return True
|
|
|
|
@staticmethod
|
|
def _open_partial(path: str) -> tuple[int, str]:
|
|
"""
|
|
Create a scratch file beside `path` and return it open for writing.
|
|
|
|
Beside it, so moving the finished download into place is a rename within one
|
|
directory. `0o666` rather than a private mode because the umask is what decided
|
|
the permissions of a downloaded file before, and still should.
|
|
"""
|
|
base = f"{path}.{os.getpid()}-{threading.get_ident()}"
|
|
for attempt in range(100):
|
|
tmp_path = f"{base}-{attempt}.mdrspart"
|
|
try:
|
|
return os.open(tmp_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666), tmp_path
|
|
except FileExistsError:
|
|
continue
|
|
except OSError as e:
|
|
raise UnexpectedException(f"Cannot write `{path}`: {e}")
|
|
raise UnexpectedException(f"Could not create a temporary file beside `{path}`.")
|
|
|
|
def _get_mime_type(self, path: str) -> str:
|
|
mt = mimetypes.guess_type(path)
|
|
if mt:
|
|
return mt[0] or self.FALLBACK_MIMETYPE
|
|
return self.FALLBACK_MIMETYPE
|