From d28bc72e211f9ff5be9d8cd8db55c77babca219e Mon Sep 17 00:00:00 2001
From: Alan De Smet <alan.desmet@ssec.wisc.edu>
Date: Mon, 16 Aug 2021 15:46:08 -0500
Subject: [PATCH] Better error messages on download problems
Especially collect up "no downloads allowed, but I'm missing files"
---
csppfetch/__init__.py | 37 +++++++++++++++++++++++++++++++++++--
1 file changed, 35 insertions(+), 2 deletions(-)
diff --git a/csppfetch/__init__.py b/csppfetch/__init__.py
index 374505a..021cfaa 100644
--- a/csppfetch/__init__.py
+++ b/csppfetch/__init__.py
@@ -283,6 +283,11 @@ def download_to_file_no_lock(url, dst, fileobj, timeout=HUGE_TIMEOUT):
with urllib.request.urlopen(url) as req:
copyfileobj(req,fileobj)
+class DownloadsDisabledException(Exception):
+ pass
+
+class DownloadsFailedException(Exception):
+ pass
def download_all(url_to_dst, download_stats, do_download, timeout):
""" Download a Dict of URL keys to their matching local values
@@ -328,6 +333,10 @@ def download_all(url_to_dst, download_stats, do_download, timeout):
dst, download_stats, do_download)
else:
todo[src] = dst
+
+ if not do_download:
+ raise DownloadsDisabledException(f"Some requested files are not present, and downloads were disabled. Files: {list(todo.values())}")
+
url_to_dst = todo
with contextlib.ExitStack() as lock_stack:
@@ -787,9 +796,21 @@ class Downloader:
base_urls = self.get_base_urls()
problems = []
+
+ def all_problems_are_missing_files(problems):
+ for problempair in problems:
+ problem, e = problempair
+ if not isinstance(e, DownloadsDisabledException):
+ return False
+ return True
+
def dump_problems(logfunc,problems):
last_problem = None
- for problem in problems:
+ if all_problems_are_missing_files(problems):
+ logfunc(" No suitable files were available locally, and downloads are disabled")
+ return
+ for problempair in problems:
+ problem = problempair[0]
if problem != last_problem:
logfunc(" "+problem)
last_problem = problem
@@ -820,13 +841,25 @@ class Downloader:
# when we already have the files!
pass
return full_urls_to_file.values()
+ except DownloadsDisabledException as e:
+ problems.append((f'Downloads disabled and one or more files are not locally available in file set {description}', e))
except Exception as e:
- problems.append(f"Failed to download file set {description}: {str(e)}")
+ problems.append((f"Failed to download file set {description}: {str(e)}", e))
if attempt != retries:
sleep(retry_wait)
+
+ if all_problems_are_missing_files(problems):
+ raise DownloadsFailedException(f"No usable {self.name} file set is present locally and downloads are disabled")
+
logging.error("Unable to download any file set, reasons include:")
dump_problems(logging.error, problems)
+ try:
+ first = problems[0][0]
+ except:
+ first = ""
+ raise DownloadsFailedException("Errors occurred during download, including "+first)
+
def download_for_time(self, time, dst, timeout=30, retries=3, retry_wait=20, do_download = True, download_stats = None):
""" Download files needed to process data a time, writing into dst
--
GitLab