diff --git a/edosl0util/cli/rdr2l0.py b/edosl0util/cli/rdr2l0.py
index d40814c7e507185f0ced477be1a1953f8d7bcbb5..061f69532566115b01d41959c1197466b44cacb2 100644
--- a/edosl0util/cli/rdr2l0.py
+++ b/edosl0util/cli/rdr2l0.py
@@ -8,6 +8,7 @@ __copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights res
import os
import glob
import logging
+import tempfile
from datetime import datetime, timedelta
from edosl0util import stream, merge, jpssrdr
@@ -16,81 +17,157 @@ from edosl0util.cli import util
LOG = logging
+satellite_to_scid = {
+ 'snpp': 157,
+}
+
+
+def pdsfilename(product, created):
+ if len(product) < 20:
+ product = product + 'A'*(20-len(product))
+ return '{}XT{:%y%j%H%M%S}001.PDS'.format(product, created)
+
+
+def remove_files(files):
+ [os.remove(f) for f in files]
+
+
+def _do_dump(filepat, product, rdrs, start, end):
+ for filepath in rdrs:
+ LOG.info('dumping %s', filepath)
+ jpssrdr.write_rdr_datasets(filepath)
+
+ # alphanumeric sorting to bootstrap final sort
+ inputs = sorted(glob.glob(filepat))
+ streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
+
+ pdsname = pdsfilename(product, start)
+ LOG.info('merging to %s', pdsname)
+ with open(pdsname, 'wb') as dest:
+ merge.merge(streams, output=dest, trunc_to=[start, end])
+ return pdsname
+
+
+def cris_hsk(satellite, rdrs, start, end):
+ product = 'P{}1280CRISHSK'.format(satellite_to_scid[satellite])
+ return _do_dump('*.telemetry.pkts', product, rdrs, start, end)
+
+
+def cris_dwell(satellite, rdrs, start, end):
+ product = 'P{}1291CRISDWELL'.format(satellite_to_scid[satellite])
+ return _do_dump('*.dwell.pkts', product, rdrs, start, end)
+
+
+def cris_sci(satellite, rdrs, start, end):
+ product = 'P{}1289CRISSCIENCE'.format(satellite_to_scid[satellite])
+ return _do_dump('*.science.pkts', product, rdrs, start, end)
+
+
+def atms_hsk(satellite, rdrs, start, end):
+ product = 'P{}0518ATMSHSK'.format(satellite_to_scid[satellite])
+ return _do_dump('*.telemetry.pkts', product, rdrs, start, end)
+
+
+def atms_dwell(satellite, rdrs, start, end):
+ product = 'P{}0517ATMSDWELL'.format(satellite_to_scid[satellite])
+ return _do_dump('*.dwell.pkts', product, rdrs, start, end)
+
+
+def atms_sci(satellite, rdrs, start, end):
+ product = 'P{}0515ATMSSCIENCE'.format(satellite_to_scid[satellite])
+ return _do_dump('*.science.pkts', product, rdrs, start, end)
+
+
+def viirs_sci(satellite, sci, start, end):
+ product = 'P{}0826VIIRSSCIENCE'.format(satellite_to_scid[satellite])
+ return _do_dump('*.science.pkts', product, rdrs, start, end)
+
+
def main():
# XXX: This currently uses standard CCSDS packet merging that does not have
# any context regarding fill packets. In the future if it is desired to
# keep fill packets a new RDR specific merge will be required.
parser = util.default_parser()
parser.description = __doc__
+ parser.add_argument('--leave-pkts', action='store_true',
+ help='Do not delete intermediate .pkts files')
parser.add_argument(
- '--minutes', type=int, default=120,
- help=('Output size in minutes. The resulting file will be truncated '
- 'to created + minutes.'))
- parser.add_argument(
- '--science', action='store_true',
- help='Dump science dataset')
- parser.add_argument(
- '--ancillary', action='store_true',
- help='Dump spacecraft ancillary datasets')
+ '-S', '--satellite', choices=['snpp'], default='snpp',
+ help='Satellite used to set SCID')
def timestamp(v):
return datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
parser.add_argument(
- '--created', type=timestamp, default=datetime.utcnow(),
- help=('Time to use for creation time (yyyy-mm-dd hh:mm:ss).'))
- parser.add_argument('rdr', nargs='+')
+ '-s', '--start', type=timestamp, required=True,
+ help=('File start time. Data before this time will be dropped. This '
+ 'time also determines the creation time in the filename. Format '
+ 'is YYYY-mm-dd HH:MM:SS.'),
+ )
+ parser.add_argument(
+ '-e', '--end', nargs='?', type=timestamp,
+ help=('File end time. Data after this time will be dropped. If omitted '
+ 'end will default to start + 2 hours. Format '
+ 'is YYYY-mm-dd HH:MM:SS.'),
+ )
+ subs = parser.add_subparsers(title='Destination level 0 data type')
+
+ def cmd_cris_hsk(args):
+ cris_hsk(args.satellite, args.rcrit, args.start, args.end)
+ subp = subs.add_parser('CRISHSK')
+ subp.add_argument('rcrit', nargs='+')
+ subp.set_defaults(func=cmd_cris_hsk)
+
+ def cmd_cris_dwell(args):
+ return cris_dwell(args.satellite, args.rdrs, args.start, args.end)
+ subp = subs.add_parser('CRISDWELL')
+ subp.add_argument(
+ 'rdrs', nargs='+',
+ help=('RCRIH, RCRIM, and RCRII files. The same number of each is '
+ 'required to produce a valid L0 file.'))
+ subp.set_defaults(func=cmd_cris_dwell)
+
+ def cmd_cris_sci(args):
+ return cris_sci(args.satellite, args.rcris, args.start, args.end)
+ subp = subs.add_parser('CRISSCIENCE')
+ subp.add_argument('rcris', nargs='+')
+ subp.set_defaults(func=cmd_cris_sci)
+
+ def cmd_atms_hsk(args):
+ return atms_hsk(args.satellite, args.ratmt, args.start, args.end)
+ subp = subs.add_parser('ATMSHSK')
+ subp.add_argument('ratmt', nargs='+')
+ subp.set_defaults(func=cmd_atms_hsk)
+
+ def cmd_atms_dwell(args):
+ return atms_dwell(args.satellite, args.rdrs, args.start, args.end)
+ subp = subs.add_parser('ATMSDWELL')
+ subp.add_argument(
+ 'rdrs', nargs='+',
+ help=('RATMW and RATMM files. The same number of each is required '
+ 'to produce a valid L0 file.'))
+ subp.set_defaults(func=cmd_atms_dwell)
+
+ def cmd_atms_sci(args):
+ return atms_sci(args.satellite, args.ratms, args.start, args.end)
+ subp = subs.add_parser('ATMSSCIENCE')
+ subp.add_argument('ratms', nargs='+')
+ subp.set_defaults(func=cmd_atms_sci)
+
+ def cmd_viirs_sci(args):
+ return viirs_sci(args.satellite, args.rvirs, args.start, args.end)
+ subp = subs.add_parser('VIIRSSCIENCE')
+ subp.add_argument('rvirs', nargs='+')
+ subp.set_defaults(func=cmd_viirs_sci)
+
args = parser.parse_args()
+
+ if not args.end:
+ args.end = args.start + timedelta(hours=2)
+
util.configure_logging(args)
- def pdsfilename(product):
- return 'P157{}XT{:%y%j%H%M%S}001.PDS'.format(product, args.created)
-
- def remove_files(files):
- [os.remove(f) for f in files]
-
- for filepath in args.rdr:
- dirname, rdrname = os.path.split(filepath)
- LOG.info("dumping %s", filepath)
- jpssrdr.write_rdr_datasets(
- filepath,
- science=args.science,
- ancillary=args.ancillary,
- skipfill=True)
-
- interval = [args.created, args.created + timedelta(minutes=args.minutes, microseconds=-1)]
- LOG.info("merge and truncate to %s", interval)
-
- if args.ancillary:
- for apid in (0, 8, 11):
- inputs = sorted(glob.glob('*.anc{:d}.pkts'.format(apid)))
- product = '{:04d}AAAAAAAAAAA'.format(apid)
- pdsname = pdsfilename(product)
- LOG.info("merging apid %d to %s", apid, pdsname)
- # there are potentially a very large number of inputs (RNSCA) so
- # make sure to keep track of files so we can explicitly close them
- files = [open(f, 'rb') for f in inputs]
- streams = [stream.jpss_packet_stream(f) for f in files]
- with open(pdsname, 'wb') as dest:
- merge.merge(streams, output=dest, trunc_to=interval)
- for f in files:
- f.close()
- remove_files(inputs)
-
- science_products = {
- 'RNSCA-RVIRS': '0826VIIRSSCIENCE',
- 'RCRIS-RNSCA': '1289CRISSCIENCEA',
- 'RATMS-RNSCA': '0515ATMSSCIENCEA'}
- if args.science:
- file_type = os.path.basename(args.rdr[0]).split('_')[0]
- if file_type not in science_products:
- parser.exit(1, '%s is not a supported science file type\n' % file_type)
- inputs = sorted(glob.glob('*.science.pkts'))
- product = science_products[file_type]
- pdsname = pdsfilename(product)
- LOG.info("merging %s to %s", product, pdsname)
- streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
- with open(pdsname, 'wb') as dest:
- merge.merge(streams, output=dest, trunc_to=interval)
- remove_files(inputs)
+ pdsname = args.func(args)
+ if not args.leave_pkts:
+ remove_files(glob.glob('*.pkts'))
if __name__ == '__main__':
diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py
index 71a8d3652b6a265d2c4a68706cbd3d421f949807..20da569eb0ff79e82058dca0d670ada447f2bfce 100644
--- a/edosl0util/jpssrdr.py
+++ b/edosl0util/jpssrdr.py
@@ -155,11 +155,37 @@ def _generate_packet_datasets(group):
yield name, np.array(ds)
+def _find_data_group(fobj, name, sensors=['viirs', 'cris', 'atms']):
+ for sensor in sensors:
+ group = fobj.get('/All_Data/{}-{}-RDR_All'.format(sensor.upper(), name.upper()))
+ if group:
+ return group
+
+
def _find_science_group(fobj):
- for sensor in ['viirs', 'cris', 'atms']:
- group = fobj.get('/All_Data/{}-SCIENCE-RDR_All'.format(sensor.upper()))
+ return _find_data_group(fobj, 'SCIENCE')
+
+
+def _find_dwell_group(fobj):
+ for name in ('HSKDWELL', 'IMDWELL', 'SSMDWELL'):
+ group = _find_data_group(fobj, name, sensors=['cris'])
if group:
return group
+ group = _find_data_group(fobj, 'DWELL', sensors=['atms'])
+ if group:
+ return group
+
+
+def _find_diag_group(fobj):
+ return _find_data_group(fobj, 'DIAGNOSTIC', sensors=['cris', 'atms'])
+
+
+def _find_telemetry_group(fobj):
+ return _find_data_group(fobj, 'TELEMETRY', sensors=['cris', 'atms'])
+
+
+def _find_spacecraft_group(fobj):
+ return _find_data_group(fobj, 'SPACESCRAFT', sensors=['cris', 'atms'])
def _rdrs_for_packet_dataset(group):
@@ -177,10 +203,13 @@ def _rdrs_for_packet_dataset(group):
def rdr_datasets(filepath):
fobj = H5File(filepath)
- rdr = {}
- rdr['science'] = _rdrs_for_packet_dataset(_find_science_group(fobj))
- rdr['ancillary'] = _rdrs_for_packet_dataset(
- fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All'))
+ rdr = dict(
+ telemetry=_rdrs_for_packet_dataset(_find_telemetry_group(fobj)),
+ diagnostic=_rdrs_for_packet_dataset(_find_diag_group(fobj)),
+ dwell=_rdrs_for_packet_dataset(_find_dwell_group(fobj)),
+ science=_rdrs_for_packet_dataset(_find_science_group(fobj)),
+ ancillary=_rdrs_for_packet_dataset(_find_spacecraft_group(fobj)),
+ )
fobj.close()
return rdr
@@ -222,26 +251,19 @@ def _write_packets(pkts, dest, skipfill):
dest.write(pkt.packet)
-def write_rdr_datasets(filepath, science=True, ancillary=True, skipfill=False):
+def write_rdr_datasets(filepath, skipfill=False):
rdrname = os.path.basename(filepath)
rdrs = rdr_datasets(filepath)
- if science:
- with open('{}.science.pkts'.format(rdrname), 'wb') as dest:
- for idx, rdr in enumerate(rdrs['science']):
+ for typekey in rdrs:
+ if not rdrs[typekey]:
+ continue
+ pktfile = '{}.{}.pkts'.format(rdrname, typekey)
+ LOG.debug('writing %s', pktfile)
+ with open(pktfile, 'wb') as dest:
+ for idx, rdr in enumerate(rdrs[typekey]):
LOG.debug(
- 'writing science gran %d %s-%s-%s',
+ '... %s gran %d %s-%s-%s', typekey,
idx, rdr.header.satellite, rdr.header.sensor, rdr.header.type_id)
_write_packets(rdr.packets(), dest, skipfill)
-
- if ancillary:
- for idx, rdr in enumerate(rdrs['ancillary']):
- packets = {a.value: rdr.packets_for_apid(a)
- for a in rdr.apids}
- for apid, pkts in packets.items():
- LOG.debug(
- 'writing ancillary gran %d %s-%s-%s %d',
- idx, rdr.header.satellite, rdr.header.sensor,
- rdr.header.type_id, apid)
- with open('{}.anc{}.pkts'.format(rdrname, apid), 'wb') as dest:
- _write_packets(pkts, dest, skipfill)
+ return rdrs
diff --git a/edosl0util/merge.py b/edosl0util/merge.py
index af4b604f1c8691c2c545cb833d9b4cc508fc91b9..c55dd67e7efa931b0ed84c8640e0803ae3ab40ec 100644
--- a/edosl0util/merge.py
+++ b/edosl0util/merge.py
@@ -42,7 +42,7 @@ class _Ptr(object):
(that.stamp, that.apid)
)
- # instances with same stamp/apid will compare the same
+ # instances with same stamp/apid/size will compare the same
def __hash__(self):
return hash((self.stamp, self.apid, self.size))
@@ -94,6 +94,9 @@ def _sort_by_time_apid(index, order=None):
def _filter_duplicates_by_size(index):
+ """
+ Take the packet with the largest size.
+ """
filtered = OrderedDict()
for ptr in index:
key = (ptr.stamp, ptr.apid)