diff --git a/edosl0util/cli/rdr2l0.py b/edosl0util/cli/rdr2l0.py index e595f1c967f8dbaa7f2ebe2aeb619b3a04382ced..d40814c7e507185f0ced477be1a1953f8d7bcbb5 100644 --- a/edosl0util/cli/rdr2l0.py +++ b/edosl0util/cli/rdr2l0.py @@ -32,11 +32,6 @@ def main(): parser.add_argument( '--ancillary', action='store_true', help='Dump spacecraft ancillary datasets') - """ - parser.add_argument( - '--skipfill', action='store_true', - help='Skip any packets marked as containing fill in the RDR') - """ def timestamp(v): return datetime.strptime(v, '%Y-%m-%d %H:%M:%S') parser.add_argument( @@ -52,17 +47,8 @@ def main(): def remove_files(files): [os.remove(f) for f in files] - science_products = { - 'RNSCA-RVIRS': '0826VIIRSSCIENCE', - 'RCRIS-RNSCA': '1289CRISSCIENCEA', - 'RATMS-RNSCA': '0515ATMSSCIENCEA'} - - file_type = os.path.basename(args.rdr[0]).split('_')[0] - LOG.info('File type %s', file_type) for filepath in args.rdr: dirname, rdrname = os.path.split(filepath) - if not rdrname.startswith(file_type): - raise ValueError('All rdrs must have same file type', rdrname) LOG.info("dumping %s", filepath) jpssrdr.write_rdr_datasets( filepath, @@ -79,11 +65,22 @@ def main(): product = '{:04d}AAAAAAAAAAA'.format(apid) pdsname = pdsfilename(product) LOG.info("merging apid %d to %s", apid, pdsname) - streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs] - merge.merge(streams, output=open(pdsname, 'wb'), trunc_to=interval) + # there are potentially a very large number of inputs (RNSCA) so + # make sure to keep track of files so we can explicitly close them + files = [open(f, 'rb') for f in inputs] + streams = [stream.jpss_packet_stream(f) for f in files] + with open(pdsname, 'wb') as dest: + merge.merge(streams, output=dest, trunc_to=interval) + for f in files: + f.close() remove_files(inputs) + science_products = { + 'RNSCA-RVIRS': '0826VIIRSSCIENCE', + 'RCRIS-RNSCA': '1289CRISSCIENCEA', + 'RATMS-RNSCA': '0515ATMSSCIENCEA'} if args.science: + file_type = os.path.basename(args.rdr[0]).split('_')[0] if file_type not in science_products: parser.exit(1, '%s is not a supported science file type\n' % file_type) inputs = sorted(glob.glob('*.science.pkts')) @@ -91,7 +88,8 @@ def main(): pdsname = pdsfilename(product) LOG.info("merging %s to %s", product, pdsname) streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs] - merge.merge(streams, output=open(pdsname, 'wb'), trunc_to=interval) + with open(pdsname, 'wb') as dest: + merge.merge(streams, output=dest, trunc_to=interval) remove_files(inputs) diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py index d27866bebff433303bea7654aad558ef1f3c482e..71a8d3652b6a265d2c4a68706cbd3d421f949807 100644 --- a/edosl0util/jpssrdr.py +++ b/edosl0util/jpssrdr.py @@ -163,6 +163,7 @@ def _find_science_group(fobj): def _rdrs_for_packet_dataset(group): + rdrs = [] if group: for name, buf in _generate_packet_datasets(group): if buf.shape[0] < c.sizeof(StaticHeader): @@ -170,16 +171,17 @@ def _rdrs_for_packet_dataset(group): continue header = StaticHeader.from_buffer(buf) apids = _read_apid_list(header, buf) - yield CommonRdr(buf, header, list(apids)) + rdrs.append(CommonRdr(buf, header, list(apids))) + return rdrs def rdr_datasets(filepath): fobj = H5File(filepath) rdr = {} - group = _find_science_group(fobj) - rdr['science'] = _rdrs_for_packet_dataset(group) - group = fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All') - rdr['ancillary'] = _rdrs_for_packet_dataset(group) + rdr['science'] = _rdrs_for_packet_dataset(_find_science_group(fobj)) + rdr['ancillary'] = _rdrs_for_packet_dataset( + fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All')) + fobj.close() return rdr @@ -234,12 +236,12 @@ def write_rdr_datasets(filepath, science=True, ancillary=True, skipfill=False): if ancillary: for idx, rdr in enumerate(rdrs['ancillary']): - packets = {a.value: rdr.packets_for_apid(a.value) + packets = {a.value: rdr.packets_for_apid(a) for a in rdr.apids} - for apid, packets in packets.items(): + for apid, pkts in packets.items(): LOG.debug( 'writing ancillary gran %d %s-%s-%s %d', idx, rdr.header.satellite, rdr.header.sensor, - rdr.header.type_id, apid.value) + rdr.header.type_id, apid) with open('{}.anc{}.pkts'.format(rdrname, apid), 'wb') as dest: - _write_packets(packets, dest) + _write_packets(pkts, dest, skipfill)