Modify RDR extracting support to include diary data.

edosl0util/cli.py

@@ -110,15 +110,23 @@ def cmd_merge():
 
 
 def cmd_rdr2l0():
+    """
+    Extract CCSDS packets from a JPSS RDR. They are extracted in the order in
+    which they are listed in the APID list and in packet tracker order, i.e.,
+    Not in time/apid order. Files will be named using the input h5 name with
+    .science.pds and .diary[0,8,11].pds appended.
+    """
     parser = _default_parser()
-    parser.add_argument('-o', '--output')
-    parser.add_argument('-f', '--skipfill', action='store_true')
-    parser.add_argument('sensor', choices=('atms', 'cris'))
+    parser.description = cmd_rdr2l0.__doc__
+    parser.add_argument(
+        '-d', '--diary', action='store_true',
+        help='Write diary[0,8,11].pds if available')
+    parser.add_argument(
+        '-f', '--skipfill', action='store_true',
+        help=('Skip any packets that are marked in the packet tracker as '
+              'containing fill'))
     parser.add_argument('rdr')
     args = parser.parse_args()
     _configure_logging(args)
 
-    output = args.output or args.rdr + '.pds'
-    with io.open(output, 'wb') as fptr:
-        for packet in jpssrdr.convert_to_nasa_l0(args.sensor, args.rdr):
-            fptr.write(packet)
+    jpssrdr.write_rdr_datasets(args.rdr, args.diary)

edosl0util/jpssrdr.py

@@ -9,8 +9,10 @@ Code for reading/writing/manipulating JPSS Common RDR files as documented in:
 """
 __copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
 
+import os
 import logging
 import ctypes as c
+from datetime import datetime
 from collections import namedtuple
 
 import numpy as np
@@ -90,13 +92,17 @@ Packet = namedtuple('Packet', ('tracker', 'packet'))
 
 
 class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
+    """
+    A single common rdr as found in a RawApplicationPackets_X dataset.
+    """
 
     def packets_for_apid(self, apid):
         """
         Return a generator that yields tuples of (tracker, packet) for the
         given `Apid`.
         """
-        return _packets_for_apid(self.buf, self.header, apid)
+        for tracker, packet in _packets_for_apid(self.buf, self.header, apid):
+            yield Packet(tracker, packet)
 
     def packets(self):
         """
@@ -104,8 +110,8 @@ class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
         apid list.
         """
         for apid in self.apids:
-            for tracker, packet in self.packets_for_apid(apid):
-                yield Packet(tracker, packet)
+            for packet in self.packets_for_apid(apid):
+                yield packet
 
 
 def _packets_for_apid(buf, header, apid):
@@ -137,31 +143,54 @@ def _read_apid_list(header, buf):
         offset += c.sizeof(Apid)
 
 
-def read_common_rdrs(sensor, filepath):
-    """
-    Generate CommonRdr-s for each dataset(granule) in `filelpath`
-    """
-    for buf in read_rdr_datasets(sensor, filepath):
-        header = StaticHeader.from_buffer(buf)
-        apids = _read_apid_list(header, buf)
-        yield CommonRdr(buf, header, list(apids))
+def _sorted_packet_dataset_names(names):
+    names = [k for k in names if k.startswith('RawApplicationPackets_')]
+    cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup))  # noqa
+    return sorted(names, cmp=cmp_names)
 
 
-def read_rdr_datasets(sensor, filepath):
-    """
-    Generate byte arrays of granule RawApplicationPackets in granule number
-    order.
-    """
-    sensor = sensor.upper()
+def _generate_packet_datasets(group):
+    dsnames = group.keys()
+    for name in _sorted_packet_dataset_names(dsnames):
+        ds = group[name]
+        yield np.array(ds)
+
+
+def _find_science_group(fobj):
+    for sensor in ['viirs', 'cris', 'atms']:
+        group = fobj.get('/All_Data/{}-SCIENCE-RDR_All'.format(sensor.upper()))
+        if group:
+            return group
+
+
+def _rdrs_for_packet_dataset(group):
+    if group:
+        for buf in _generate_packet_datasets(group):
+            header = StaticHeader.from_buffer(buf)
+            apids = _read_apid_list(header, buf)
+            yield CommonRdr(buf, header, list(apids))
+
+
+def rdr_datasets(filepath):
     fobj = H5File(filepath)
-    grp = fobj['/All_Data/{}-SCIENCE-RDR_All'.format(sensor)]
+    rdr = {}
+    group = _find_science_group(fobj)
+    rdr['science'] = _rdrs_for_packet_dataset(group)
+    group = fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All')
+    rdr['diary'] = _rdrs_for_packet_dataset(group)
+    return rdr
 
-    dsnames = [k for k in grp.keys() if k.startswith('RawApplicationPackets_')]
-    # compare ds names by index after '_'
-    cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup))  # noqa
-    for name in sorted(dsnames, cmp=cmp_names):
-        ds = grp[name]
-        yield np.array(ds)
+
+def rdr_ccsds_packets(rdr, skipfill=False):
+    """
+    Get all CCSDS packets from an `CommonRdr` in packet tracker order.
+    """
+    for packet in rdr.packets():
+        if packet.tracker.fill_percent != 0:
+            LOG.debug('fill: %s', packet.tracker)
+            if skipfill:
+                continue
+        yield packet.packet
 
 
 def sort_packets_by_obs_time(packets):
@@ -182,17 +211,25 @@ def sort_packets_by_apid(packets, order=None):
         return sorted(packets, key=lambda p: p.packet.apid)
 
 
-def convert_to_nasa_l0(sensor, filename, skipfill=False):
-    """
-    Convert a JSPP RDR to a NASA L0 PDS file.
-    """
-    for idx, rdr in enumerate(read_common_rdrs(sensor, filename)):
-        LOG.debug(rdr.header)
-        for apid in rdr.apids:
-            LOG.debug(apid)
-        for packet in rdr.packets():
-            if packet.tracker.fill_percent != 0:
-                LOG.debug('fill: %s', packet.tracker)
-                if skipfill:
+def write_rdr_datasets(filepath, diary=True, skipfill=False):
+    def write_packets(pkts, dest):
+        with open(dest, 'wb') as fptr:
+            for pkt in pkts:
+                if pkt.tracker.fill_percent != 0 and skipfill:
                     continue
-            yield packet.packet
+                fptr.write(pkt.packet)
+    rdrname = os.path.basename(filepath)
+    rdrs = rdr_datasets(filepath)
+    for idx, rdr in enumerate(rdrs['science']):
+        fname = '{}.science.pds'.format(rdrname)
+        hdr = rdr.header
+        LOG.debug('writing gran %d %s-%s-%s', idx, hdr.satellite, hdr.sensor, hdr.type_id)
+        write_packets(rdr.packets(), fname)
+
+    if diary:
+        for idx, rdr in enumerate(rdrs['diary']):
+            for apid in rdr.apids:
+                fname = '{}.diary{}.pds'.format(rdrname, apid.value)
+                hdr = rdr.header
+                LOG.debug('writing gran %d %s-%s-%s %d', idx, hdr.satellite, hdr.sensor, hdr.type_id, apid.value)
+                write_packets(rdr.packets_for_apid(apid), fname)