diff --git a/csppfetch/__init__.py b/csppfetch/__init__.py index 374505af83df3c83dd4bbf1eea18b2cea408c9a1..021cfaa089771127d1d2d4590c0bd64274fc3142 100644 --- a/csppfetch/__init__.py +++ b/csppfetch/__init__.py @@ -283,6 +283,11 @@ def download_to_file_no_lock(url, dst, fileobj, timeout=HUGE_TIMEOUT): with urllib.request.urlopen(url) as req: copyfileobj(req,fileobj) +class DownloadsDisabledException(Exception): + pass + +class DownloadsFailedException(Exception): + pass def download_all(url_to_dst, download_stats, do_download, timeout): """ Download a Dict of URL keys to their matching local values @@ -328,6 +333,10 @@ def download_all(url_to_dst, download_stats, do_download, timeout): dst, download_stats, do_download) else: todo[src] = dst + + if not do_download: + raise DownloadsDisabledException(f"Some requested files are not present, and downloads were disabled. Files: {list(todo.values())}") + url_to_dst = todo with contextlib.ExitStack() as lock_stack: @@ -787,9 +796,21 @@ class Downloader: base_urls = self.get_base_urls() problems = [] + + def all_problems_are_missing_files(problems): + for problempair in problems: + problem, e = problempair + if not isinstance(e, DownloadsDisabledException): + return False + return True + def dump_problems(logfunc,problems): last_problem = None - for problem in problems: + if all_problems_are_missing_files(problems): + logfunc(" No suitable files were available locally, and downloads are disabled") + return + for problempair in problems: + problem = problempair[0] if problem != last_problem: logfunc(" "+problem) last_problem = problem @@ -820,13 +841,25 @@ class Downloader: # when we already have the files! pass return full_urls_to_file.values() + except DownloadsDisabledException as e: + problems.append((f'Downloads disabled and one or more files are not locally available in file set {description}', e)) except Exception as e: - problems.append(f"Failed to download file set {description}: {str(e)}") + problems.append((f"Failed to download file set {description}: {str(e)}", e)) if attempt != retries: sleep(retry_wait) + + if all_problems_are_missing_files(problems): + raise DownloadsFailedException(f"No usable {self.name} file set is present locally and downloads are disabled") + logging.error("Unable to download any file set, reasons include:") dump_problems(logging.error, problems) + try: + first = problems[0][0] + except: + first = "" + raise DownloadsFailedException("Errors occurred during download, including "+first) + def download_for_time(self, time, dst, timeout=30, retries=3, retry_wait=20, do_download = True, download_stats = None): """ Download files needed to process data a time, writing into dst