From d11af0e7fcc84e7d14269b684da45ca04757e330 Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Thu, 12 Nov 2015 01:17:12 +0000
Subject: [PATCH] Fixs rdr2l0 CLI tool.
* Don't require same file type for all files.
* Fix bugs in packet handling.
* Properly close files to avoid exhausting file descriptors for large
numbers of RNSCA flies.
---
edosl0util/cli/rdr2l0.py | 32 +++++++++++++++-----------------
edosl0util/jpssrdr.py | 20 +++++++++++---------
2 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/edosl0util/cli/rdr2l0.py b/edosl0util/cli/rdr2l0.py
index e595f1c..d40814c 100644
--- a/edosl0util/cli/rdr2l0.py
+++ b/edosl0util/cli/rdr2l0.py
@@ -32,11 +32,6 @@ def main():
parser.add_argument(
'--ancillary', action='store_true',
help='Dump spacecraft ancillary datasets')
- """
- parser.add_argument(
- '--skipfill', action='store_true',
- help='Skip any packets marked as containing fill in the RDR')
- """
def timestamp(v):
return datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
parser.add_argument(
@@ -52,17 +47,8 @@ def main():
def remove_files(files):
[os.remove(f) for f in files]
- science_products = {
- 'RNSCA-RVIRS': '0826VIIRSSCIENCE',
- 'RCRIS-RNSCA': '1289CRISSCIENCEA',
- 'RATMS-RNSCA': '0515ATMSSCIENCEA'}
-
- file_type = os.path.basename(args.rdr[0]).split('_')[0]
- LOG.info('File type %s', file_type)
for filepath in args.rdr:
dirname, rdrname = os.path.split(filepath)
- if not rdrname.startswith(file_type):
- raise ValueError('All rdrs must have same file type', rdrname)
LOG.info("dumping %s", filepath)
jpssrdr.write_rdr_datasets(
filepath,
@@ -79,11 +65,22 @@ def main():
product = '{:04d}AAAAAAAAAAA'.format(apid)
pdsname = pdsfilename(product)
LOG.info("merging apid %d to %s", apid, pdsname)
- streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
- merge.merge(streams, output=open(pdsname, 'wb'), trunc_to=interval)
+ # there are potentially a very large number of inputs (RNSCA) so
+ # make sure to keep track of files so we can explicitly close them
+ files = [open(f, 'rb') for f in inputs]
+ streams = [stream.jpss_packet_stream(f) for f in files]
+ with open(pdsname, 'wb') as dest:
+ merge.merge(streams, output=dest, trunc_to=interval)
+ for f in files:
+ f.close()
remove_files(inputs)
+ science_products = {
+ 'RNSCA-RVIRS': '0826VIIRSSCIENCE',
+ 'RCRIS-RNSCA': '1289CRISSCIENCEA',
+ 'RATMS-RNSCA': '0515ATMSSCIENCEA'}
if args.science:
+ file_type = os.path.basename(args.rdr[0]).split('_')[0]
if file_type not in science_products:
parser.exit(1, '%s is not a supported science file type\n' % file_type)
inputs = sorted(glob.glob('*.science.pkts'))
@@ -91,7 +88,8 @@ def main():
pdsname = pdsfilename(product)
LOG.info("merging %s to %s", product, pdsname)
streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
- merge.merge(streams, output=open(pdsname, 'wb'), trunc_to=interval)
+ with open(pdsname, 'wb') as dest:
+ merge.merge(streams, output=dest, trunc_to=interval)
remove_files(inputs)
diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py
index d27866b..71a8d36 100644
--- a/edosl0util/jpssrdr.py
+++ b/edosl0util/jpssrdr.py
@@ -163,6 +163,7 @@ def _find_science_group(fobj):
def _rdrs_for_packet_dataset(group):
+ rdrs = []
if group:
for name, buf in _generate_packet_datasets(group):
if buf.shape[0] < c.sizeof(StaticHeader):
@@ -170,16 +171,17 @@ def _rdrs_for_packet_dataset(group):
continue
header = StaticHeader.from_buffer(buf)
apids = _read_apid_list(header, buf)
- yield CommonRdr(buf, header, list(apids))
+ rdrs.append(CommonRdr(buf, header, list(apids)))
+ return rdrs
def rdr_datasets(filepath):
fobj = H5File(filepath)
rdr = {}
- group = _find_science_group(fobj)
- rdr['science'] = _rdrs_for_packet_dataset(group)
- group = fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All')
- rdr['ancillary'] = _rdrs_for_packet_dataset(group)
+ rdr['science'] = _rdrs_for_packet_dataset(_find_science_group(fobj))
+ rdr['ancillary'] = _rdrs_for_packet_dataset(
+ fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All'))
+ fobj.close()
return rdr
@@ -234,12 +236,12 @@ def write_rdr_datasets(filepath, science=True, ancillary=True, skipfill=False):
if ancillary:
for idx, rdr in enumerate(rdrs['ancillary']):
- packets = {a.value: rdr.packets_for_apid(a.value)
+ packets = {a.value: rdr.packets_for_apid(a)
for a in rdr.apids}
- for apid, packets in packets.items():
+ for apid, pkts in packets.items():
LOG.debug(
'writing ancillary gran %d %s-%s-%s %d',
idx, rdr.header.satellite, rdr.header.sensor,
- rdr.header.type_id, apid.value)
+ rdr.header.type_id, apid)
with open('{}.anc{}.pkts'.format(rdrname, apid), 'wb') as dest:
- _write_packets(packets, dest)
+ _write_packets(pkts, dest, skipfill)
--
GitLab