Skip to content
Snippets Groups Projects
Commit c377588a authored by Alan De Smet's avatar Alan De Smet
Browse files

Progress.finished_file get relative path, not absolute

Relative to the dst passed to Downloader's update_cache/download_for_time
parent 79ea3e8c
No related branches found
No related tags found
No related merge requests found
...@@ -178,13 +178,14 @@ class ProgressDisabled(Progress): ...@@ -178,13 +178,14 @@ class ProgressDisabled(Progress):
class DownloadStatistics: class DownloadStatistics:
def __init__(self, progress = None): def __init__(self, progress = None, dstroot = ""):
self.downloaded_size = 0 self.downloaded_size = 0
self.downloaded_files = set() self.downloaded_files = set()
self.cached_size = 0 self.cached_size = 0
self.cached_files = set() self.cached_files = set()
self.start = datetime.datetime.now() self.start = datetime.datetime.now()
self.end = None self.end = None
self.dstroot = dstroot
self.max_attempts = 0 self.max_attempts = 0
self.current_attempt = 0 self.current_attempt = 0
...@@ -245,7 +246,8 @@ class DownloadStatistics: ...@@ -245,7 +246,8 @@ class DownloadStatistics:
logging.debug(f"{dst} downloaded ({state}).") logging.debug(f"{dst} downloaded ({state}).")
self.downloaded_size += size self.downloaded_size += size
self.downloaded_files.add(dst) self.downloaded_files.add(dst)
self.progress.finished_file(self, dst, is_cache_hit=False) reldst = os.path.relpath(dst, self.dstroot)
self.progress.finished_file(self, reldst, is_cache_hit=False)
def download_already_present(self, state, dst): def download_already_present(self, state, dst):
logging.debug("{0} already exists ({1}). Skipping.".format(dst, state)) logging.debug("{0} already exists ({1}). Skipping.".format(dst, state))
...@@ -256,7 +258,8 @@ class DownloadStatistics: ...@@ -256,7 +258,8 @@ class DownloadStatistics:
if not self.already_have(dst): if not self.already_have(dst):
self.cached_files.add(dst) self.cached_files.add(dst)
self.cached_size += os.path.getsize(dst) self.cached_size += os.path.getsize(dst)
self.progress.finished_file(self, dst, is_cache_hit=True) reldst = os.path.relpath(dst, self.dstroot)
self.progress.finished_file(self, reldst, is_cache_hit=True)
def set_current_attempt(self, attempt_num): def set_current_attempt(self, attempt_num):
self.current_attempt = attempt_num self.current_attempt = attempt_num
...@@ -958,7 +961,7 @@ class Downloader: ...@@ -958,7 +961,7 @@ class Downloader:
retries = self.get_retries(retries) retries = self.get_retries(retries)
else: else:
retries = 1 retries = 1
stats = DownloadStatistics(progress=progress) stats = DownloadStatistics(progress=progress, dstroot=dst)
self.download_first_available(filesets, dst, timeout=timeout, retries=retries, retry_wait=retry_wait, do_download = do_download, download_stats = stats) self.download_first_available(filesets, dst, timeout=timeout, retries=retries, retry_wait=retry_wait, do_download = do_download, download_stats = stats)
stats.finish() stats.finish()
return stats return stats
...@@ -987,7 +990,7 @@ class Downloader: ...@@ -987,7 +990,7 @@ class Downloader:
:return: csppfetch.DownloadStatistics :return: csppfetch.DownloadStatistics
""" """
download_stats = DownloadStatistics(progress=progress) download_stats = DownloadStatistics(progress=progress, dstroot=dst)
if end_time is not None and start_time is not None: if end_time is not None and start_time is not None:
time = self._nearest_preceeding_time(end_time) time = self._nearest_preceeding_time(end_time)
if time < start_time: if time < start_time:
... ...
......
...@@ -37,6 +37,6 @@ class Progress(csppfetch.Progress): ...@@ -37,6 +37,6 @@ class Progress(csppfetch.Progress):
reason = "downloaded" reason = "downloaded"
if is_cache_hit: if is_cache_hit:
reason = "found in local cache" reason = "found in local cache"
sys.stderr.write(f" {os.path.basename(local_path)} {reason}. (Downloaded {len(stats.downloaded_files)} files, {human_bytes(stats.downloaded_size)}. Found in cache {len(stats.cached_files)} files, {human_bytes(stats.cached_size)})\n") sys.stderr.write(f" {local_path} {reason}. (Downloaded {len(stats.downloaded_files)} files, {human_bytes(stats.downloaded_size)}. Found in cache {len(stats.cached_files)} files, {human_bytes(stats.cached_size)})\n")
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment