Skip to content
Snippets Groups Projects
Commit e5baea0d authored by Ray Garcia's avatar Ray Garcia :scream_cat:
Browse files

add --quiet, which suppresses reporting if all looks good

parent 67164889
No related branches found
No related tags found
No related merge requests found
...@@ -25,6 +25,8 @@ from collections import defaultdict ...@@ -25,6 +25,8 @@ from collections import defaultdict
from datetime import datetime, timedelta from datetime import datetime, timedelta
import numpy as np import numpy as np
MIN_SUCCESSFUL_TILESETS = 2000
try: try:
import netCDF4 as nc4 import netCDF4 as nc4
except ImportError: except ImportError:
...@@ -124,8 +126,9 @@ class TileChecks(object): ...@@ -124,8 +126,9 @@ class TileChecks(object):
l1b_invalid = None l1b_invalid = None
count_by_scene = None count_by_scene = None
tile_latency_by_scene = None tile_latency_by_scene = None
quiet = None
def __init__(self, path_cache_seq=None, path_patterns=None): def __init__(self, path_cache_seq=None, path_patterns=None, quiet=None):
self.groups_already_seen = set() self.groups_already_seen = set()
self.groups_missing_tiles = {} self.groups_missing_tiles = {}
self.groups_intact = 0 self.groups_intact = 0
...@@ -135,6 +138,7 @@ class TileChecks(object): ...@@ -135,6 +138,7 @@ class TileChecks(object):
self.l1b_invalid = set() self.l1b_invalid = set()
self.count_by_scene = defaultdict(int) self.count_by_scene = defaultdict(int)
self.tile_latency_by_scene = defaultdict(list) self.tile_latency_by_scene = defaultdict(list)
self.quiet = quiet
pc = set() if path_cache_seq is None else set(path_cache_seq) pc = set() if path_cache_seq is None else set(path_cache_seq)
if path_patterns: if path_patterns:
for path_pattern in path_patterns: for path_pattern in path_patterns:
...@@ -321,27 +325,29 @@ class TileChecks(object): ...@@ -321,27 +325,29 @@ class TileChecks(object):
from pprint import pprint from pprint import pprint
intact = self.groups_intact intact = self.groups_intact
total = self.groups_total total = self.groups_total
print("{:d} of {:d} tile sets fully intact.".format(intact, total)) success = intact == total and total > MIN_SUCCESSFUL_TILESETS and not self.l1b_without_tiles and not self.l1b_invalid
if intact != total: if not (self.quiet and success):
print("tile sets with missing tiles:") print("{:d} of {:d} tile sets fully intact.".format(intact, total))
for group, (count, total) in self.groups_missing_tiles.items(): if intact != total:
print("{:03d}/{:03d} present for {}".format(count, total, group)) print("tile sets with missing tiles:")
if self.l1b_total > 0: for group, (count, total) in self.groups_missing_tiles.items():
print("{:d} of {:d} L1b radiance files had tile sets present" print("{:03d}/{:03d} present for {}".format(count, total, group))
.format(self.l1b_total - len(self.l1b_without_tiles), self.l1b_total)) if self.l1b_total > 0:
if self.l1b_invalid: print("{:d} of {:d} L1b radiance files had tile sets present"
print("{:d} invalid L1b files encountered".format(len(self.l1b_invalid))) .format(self.l1b_total - len(self.l1b_without_tiles), self.l1b_total))
pprint(list(sorted(self.l1b_invalid))) if self.l1b_invalid:
print("count by scene:") print("{:d} invalid L1b files encountered".format(len(self.l1b_invalid)))
for k, v in self.count_by_scene.items(): pprint(list(sorted(self.l1b_invalid)))
print('{}: {}'.format(k, v)) print("count by scene:")
if self.tile_latency_by_scene: for k, v in self.count_by_scene.items():
print("tileset latency stats for L1b content:") print('{}: {}'.format(k, v))
for k, v in self.tile_latency_by_scene.items(): if self.tile_latency_by_scene:
stats = latency_stats(v) print("tileset latency stats for L1b content:")
stats_str = ' '.join('{}: {:.3f}'.format(k, v) for k, v in stats.items()) for k, v in self.tile_latency_by_scene.items():
print('{}: {}'.format(k, stats_str)) stats = latency_stats(v)
return 0 if intact == total and not self.l1b_without_tiles and not self.l1b_invalid else 1 stats_str = ' '.join('{}: {:.3f}'.format(k, v) for k, v in stats.items())
print('{}: {}'.format(k, stats_str))
return 0 if success else 1
def _debug(typ_, value, tb): def _debug(typ_, value, tb):
...@@ -373,6 +379,8 @@ def main(): ...@@ -373,6 +379,8 @@ def main():
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG') help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG')
parser.add_argument('-d', '--debug', dest='debug', action='store_true', parser.add_argument('-d', '--debug', dest='debug', action='store_true',
help="enable interactive PDB debugger on exception") help="enable interactive PDB debugger on exception")
parser.add_argument('-q', '--quiet', dest='quiet', action='store_true',
help="enable interactive PDB debugger on exception")
parser.add_argument('inputs', nargs='*', parser.add_argument('inputs', nargs='*',
help="input files, file patterns, or file-list-fifos to process") help="input files, file patterns, or file-list-fifos to process")
args = parser.parse_args() args = parser.parse_args()
...@@ -381,7 +389,7 @@ def main(): ...@@ -381,7 +389,7 @@ def main():
if args.debug: if args.debug:
sys.excepthook = _debug sys.excepthook = _debug
if not args.inputs: if not args.inputs:
tc = TileChecks() tc = TileChecks(quiet=args.quiet)
if not os.isatty(0): if not os.isatty(0):
tc.tile_inventory(tc.filter_l1b_to_tiles(line.strip() for line in sys.stdin.readlines())) tc.tile_inventory(tc.filter_l1b_to_tiles(line.strip() for line in sys.stdin.readlines()))
return tc.report() return tc.report()
...@@ -390,11 +398,11 @@ def main(): ...@@ -390,11 +398,11 @@ def main():
print(__doc__) print(__doc__)
return 1 return 1
elif all_are_files(args.inputs): # single file, track down its family elif all_are_files(args.inputs): # single file, track down its family
tc = TileChecks() tc = TileChecks(quiet=args.quiet)
tc.tile_inventory(tc.filter_l1b_to_tiles(list(args.inputs))) tc.tile_inventory(tc.filter_l1b_to_tiles(list(args.inputs)))
return tc.report() return tc.report()
else: # glob wildcards, pipes with lists of files, and/or multiple files, check everything that matches else: # glob wildcards, pipes with lists of files, and/or multiple files, check everything that matches
tc = TileChecks(path_patterns=list(args.inputs)) # limit it to files matching wildcards, e.g. '*_s201924812*nc' tc = TileChecks(path_patterns=list(args.inputs), quiet=args.quiet) # limit it to files matching wildcards, e.g. '*_s201924812*nc'
tc.tile_inventory(tc.filter_l1b_to_tiles(tc.path_cache)) tc.tile_inventory(tc.filter_l1b_to_tiles(tc.path_cache))
return tc.report() return tc.report()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment