fix: clear the type checker and linter findings
flake8 reported 49 findings and pyright 10, and two of them were real bugs rather than matters of style. The rest were unused imports and four functions over the complexity limit. - Unlock a DOI folder with the id the DOI response carries, rather than an attribute the model does not have, which raised `AttributeError` on every locked DOI folder. - Size the `ls` Size column from the sub-folder on the row rather than from its parent, which pushed the later columns out of line. - Share the path resolution and the destination checks between `cp` and `mv`, and split the recursive download and the `ls` row printing, bringing all four functions under the complexity limit. - Accept a client built without a connection, which `config` and `version` rely on, and report the reason if one is then asked for. - Declare the config protocol's constructor for the type checker alone, so the protocol keeps its guard against being instantiated. - Remove 41 unused imports, and let flake8 accept black's spacing. - Point pyright at the project's own environment, without which it resolved no dependency and reported 124 findings that were not real. - Cover the two fixes and the shared `cp`/`mv` paths with new tests.
This commit is contained in:
+83
-68
@@ -154,86 +154,101 @@ class Downloader:
|
||||
) -> bool:
|
||||
"""Fetch what the remote path names, and report whether every file arrived."""
|
||||
excludes_clean = excludes or []
|
||||
# Detect DOI path: "remote:10.xxxx/prefix.ID[/optional/sub/path]"
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
|
||||
# "remote:10.xxxx/prefix.ID[/optional/sub/path]" names a published dataset rather
|
||||
# than a path within a laboratory, and is resolved through the DOI instead.
|
||||
path_component = remote_path.split(":", 1)[1] if ":" in remote_path else ""
|
||||
if self.client.is_doi(path_component):
|
||||
remote, doi, subpath = self.client.parse_doi_remote_host(remote_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
doi_folder, laboratory = self.client.find_folder_by_doi(doi, password)
|
||||
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
folder = doi_folder
|
||||
is_folder = True
|
||||
else:
|
||||
r_dirname = os.path.dirname(subpath_clean)
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_path = doi_folder.path.rstrip("/") + r_dirname
|
||||
r_parent_folder = self.client.find_folder(laboratory, abs_path, password)
|
||||
r_parent_files = self.client.find_files(r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if self.__check_excludes(excludes_clean, laboratory, r_parent_folder, file):
|
||||
return True
|
||||
context = DownloadContext(is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
return self.__multiple_download(context)
|
||||
else:
|
||||
folder_simple = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder_simple is None:
|
||||
raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
|
||||
folder = FoldersApi(self.client.connection).retrieve(folder_simple.id)
|
||||
is_folder = True
|
||||
|
||||
# For a DOI target the whole folder is the download target.
|
||||
if not is_recursive:
|
||||
# Non-recursive: download only the files at the top level of the DOI folder.
|
||||
files = self.client.find_files(folder.id)
|
||||
context = DownloadContext(is_skip_if_exists, [])
|
||||
for file in files:
|
||||
if self.__check_excludes(excludes_clean, laboratory, folder, file):
|
||||
continue
|
||||
l_path = os.path.join(l_dirname, file.name)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
return self.__multiple_download(context)
|
||||
folder_api = FoldersApi(self.client.connection)
|
||||
return self.__multiple_download_pickup_recursive_files(
|
||||
folder_api, laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
|
||||
return self.__download_doi(
|
||||
remote_path, l_dirname, is_recursive, is_skip_if_exists, password, excludes_clean
|
||||
)
|
||||
|
||||
remote, laboratory_name, r_path = self.client.parse_remote_host_with_path(remote_path)
|
||||
r_path = r_path.rstrip("/")
|
||||
r_dirname = os.path.dirname(r_path)
|
||||
r_basename = os.path.basename(r_path)
|
||||
|
||||
l_dirname = os.path.realpath(local_path)
|
||||
if not os.path.isdir(l_dirname):
|
||||
raise IllegalArgumentException(f"Local directory `{local_path}` not found.")
|
||||
laboratory = self.client.find_laboratory(laboratory_name)
|
||||
r_parent_folder = self.client.find_folder(laboratory, r_dirname, password)
|
||||
r_parent_folder = self.client.find_folder(laboratory, os.path.dirname(r_path), password)
|
||||
r_parent_files = self.client.find_files(r_parent_folder.id)
|
||||
file = find_file(r_parent_files, r_basename)
|
||||
if file is not None:
|
||||
if self.__check_excludes(excludes_clean, laboratory, r_parent_folder, file):
|
||||
return True
|
||||
context = DownloadContext(is_skip_if_exists, [])
|
||||
l_path = os.path.join(l_dirname, r_basename)
|
||||
context.files.append(DownloadFileInfo(file, l_path))
|
||||
return self.__multiple_download(context)
|
||||
else:
|
||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder is None:
|
||||
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
folder_api = FoldersApi(self.client.connection)
|
||||
return self.__multiple_download_pickup_recursive_files(
|
||||
folder_api, laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
|
||||
return self.__download_one(
|
||||
excludes_clean, laboratory, r_parent_folder, file, l_dirname, r_basename, is_skip_if_exists
|
||||
)
|
||||
folder = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder is None:
|
||||
raise IllegalArgumentException(f"File or folder `{r_path}` not found.")
|
||||
if not is_recursive:
|
||||
raise IllegalArgumentException(f"Cannot download `{r_path}`: Is a folder.")
|
||||
return self.__multiple_download_pickup_recursive_files(
|
||||
FoldersApi(self.client.connection), laboratory, folder.id, l_dirname, excludes_clean, is_skip_if_exists
|
||||
)
|
||||
|
||||
def __download_doi(
|
||||
self,
|
||||
remote_path: str,
|
||||
l_dirname: str,
|
||||
is_recursive: bool,
|
||||
is_skip_if_exists: bool,
|
||||
password: str | None,
|
||||
excludes: list[str],
|
||||
) -> bool:
|
||||
"""Fetch what a DOI names: the dataset's folder, or something inside it."""
|
||||
remote, doi, subpath = self.client.parse_doi_remote_host(remote_path)
|
||||
doi_folder, laboratory = self.client.find_folder_by_doi(doi, password)
|
||||
|
||||
subpath_clean = subpath.rstrip("/")
|
||||
if not subpath_clean:
|
||||
folder = doi_folder
|
||||
else:
|
||||
r_basename = os.path.basename(subpath_clean)
|
||||
abs_path = doi_folder.path.rstrip("/") + os.path.dirname(subpath_clean)
|
||||
r_parent_folder = self.client.find_folder(laboratory, abs_path, password)
|
||||
file = find_file(self.client.find_files(r_parent_folder.id), r_basename)
|
||||
if file is not None:
|
||||
return self.__download_one(
|
||||
excludes, laboratory, r_parent_folder, file, l_dirname, r_basename, is_skip_if_exists
|
||||
)
|
||||
folder_simple = r_parent_folder.find_sub_folder(r_basename)
|
||||
if folder_simple is None:
|
||||
raise IllegalArgumentException(f"File or folder `{subpath_clean}` not found.")
|
||||
folder = FoldersApi(self.client.connection).retrieve(folder_simple.id)
|
||||
|
||||
if is_recursive:
|
||||
return self.__multiple_download_pickup_recursive_files(
|
||||
FoldersApi(self.client.connection), laboratory, folder.id, l_dirname, excludes, is_skip_if_exists
|
||||
)
|
||||
# Without -r the dataset's own files are fetched, and its sub-folders are not.
|
||||
context = DownloadContext(is_skip_if_exists, [])
|
||||
for file in self.client.find_files(folder.id):
|
||||
if self.__check_excludes(excludes, laboratory, folder, file):
|
||||
continue
|
||||
context.files.append(DownloadFileInfo(file, os.path.join(l_dirname, file.name)))
|
||||
return self.__multiple_download(context)
|
||||
|
||||
def __download_one(
|
||||
self,
|
||||
excludes: list[str],
|
||||
laboratory: Laboratory,
|
||||
folder: Folder,
|
||||
file: File,
|
||||
l_dirname: str,
|
||||
local_name: str,
|
||||
is_skip_if_exists: bool,
|
||||
) -> bool:
|
||||
"""
|
||||
Fetch a single named file into the local directory.
|
||||
|
||||
Saved under the name the caller asked for rather than the one the server holds:
|
||||
the two are matched case-insensitively, so they need not be spelled alike.
|
||||
"""
|
||||
if self.__check_excludes(excludes, laboratory, folder, file):
|
||||
return True
|
||||
context = DownloadContext(is_skip_if_exists, [])
|
||||
context.files.append(DownloadFileInfo(file, os.path.join(l_dirname, local_name)))
|
||||
return self.__multiple_download(context)
|
||||
|
||||
def __multiple_download_pickup_recursive_files(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user