From 1828f1f1318bca785224e647113a5bc6267cb160 Mon Sep 17 00:00:00 2001
From: Alan De Smet <alan.desmet@ssec.wisc.edu>
Date: Thu, 2 Sep 2021 21:08:10 +0000
Subject: [PATCH] Downloader.download_for_time returns new DownloadStatistics
#5
---
csppfetch/__init__.py | 13 +++++++------
csppfetch/test.py | 6 ++----
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/csppfetch/__init__.py b/csppfetch/__init__.py
index e55759f..2339de1 100644
--- a/csppfetch/__init__.py
+++ b/csppfetch/__init__.py
@@ -884,15 +884,12 @@ class Downloader:
first = ""
raise DownloadsFailedException("Errors occurred during download, including "+first)
- def download_for_time(self, time, dst, download_stats, timeout=30, retries=3, retry_wait=20, do_download = True):
+ def download_for_time(self, time, dst, timeout=30, retries=3, retry_wait=20, do_download = True):
""" Download files needed to process data a time, writing into dst
If a desired file is already present, the download of that file is
considered a success and not re-downloaded.
- :param csppfetch.DownloadStatistics download_stats: DownloadStatistics
- object to collect
-
:param float timeout: seconds to allow a given download to try before
abandoning it as failed.
@@ -908,7 +905,7 @@ class Downloader:
If False, if the desired files are not already present, it's a
failure
- :return: list of strings to downloaded files
+ :return: csppfetch.DownloadStatistics
**timeout**, **retries**, and **retry_wait** are overriddeen by environment
variables ``package_env_id+TIMEOUT/RETRIES/RETRY_WAIT``
@@ -923,7 +920,9 @@ class Downloader:
retries = self.get_retries(retries)
else:
retries = 1
- return self.download_first_available(filesets, dst, timeout=timeout, retries=retries, retry_wait=retry_wait, do_download = do_download, download_stats = download_stats)
+ stats = DownloadStatistics()
+ self.download_first_available(filesets, dst, timeout=timeout, retries=retries, retry_wait=retry_wait, do_download = do_download, download_stats = stats)
+ return stats
def update_cache(self, dst, start_time=None, end_time=None, timeout=30):
@@ -941,6 +940,8 @@ class Downloader:
:param float timeout: seconds to allow a given download to try before
abandoning it as failed. Can be overriden by environment variable
package_env_id+"TIMEOUT"
+
+ :return: csppfetch.DownloadStatistics
"""
download_stats = DownloadStatistics()
diff --git a/csppfetch/test.py b/csppfetch/test.py
index 95d80df..c6a79b4 100755
--- a/csppfetch/test.py
+++ b/csppfetch/test.py
@@ -586,8 +586,7 @@ class DownloaderTests(DTestCase):
test_datetime = self.safe_test_date()
outfile = "avhrr-only-v2.20190527.nc"
with TemporaryDirectory() as dirpath:
- stats = csppfetch.DownloadStatistics()
- d.download_for_time(test_datetime, dirpath, stats) # do_download = True
+ stats = d.download_for_time(test_datetime, dirpath) # do_download = True
self.assertFileExists(dirpath+"/"+outfile)
def test_download_for_time_404(self):
@@ -598,8 +597,7 @@ class DownloaderTests(DTestCase):
with TemporaryDirectory() as dirpath:
with self.assertRaises(csppfetch.DownloadsFailedException) as raised:
with self.assertLogs(level='WARNING') as logs:
- stats = csppfetch.DownloadStatistics()
- d.download_for_time(test_datetime, dirpath, stats, retries=1) # do_download = True
+ stats = d.download_for_time(test_datetime, dirpath, retries=1) # do_download = True
self.assertFileDoesNotExist(dirpath+"/"+outfile)
self.assertRegex(logs.output[0], r'^WARNING:root:Unable to download any file set')
self.assertRegex(logs.output[1], r'HTTP Error 404')
--
GitLab