Add jpssrdr module

edosl0util/jpssrdr.py

+"""
+Code for reading/writing/manipulating JPSS Common RDR files as documented in:
+
+    Joint Polar Satellite System (JPSS)
+    Common Data Format Control Book - External (CDFCB-X)
+    Volume II - RDR Formats
+
+    http://jointmission.gsfc.nasa.gov/sciencedocs/2015-06/474-00001-02_JPSS-CDFCB-X-Vol-II_0123B.pdf
+"""
+import logging
+import ctypes as c
+from collections import namedtuple
+
+import numpy as np
+from h5py import File as H5File
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseStruct(c.BigEndianStructure):
+    _pack_ = 1
+    _fields_ = []
+
+    def __repr__(self):
+        attrs = ' '.join('%s=%s' % (f[0], getattr(self, f[0])) for f in self._fields_)
+        return '<{} {}>'.format(self.__class__.__name__, attrs)
+
+
+class StaticHeader(BaseStruct):
+
+    _fields_ = [
+        ('satellite', c.c_char * 4),
+        ('sensor', c.c_char * 16),
+        ('type_id', c.c_char * 16),
+        ('num_apids', c.c_uint32),
+        ('apid_list_offset', c.c_uint32),
+        ('pkt_tracker_offset', c.c_uint32),
+        ('ap_storage_offset', c.c_uint32),
+        ('next_pkt_pos', c.c_uint32),
+        ('start_boundry', c.c_uint64),
+        ('end_boundry', c.c_uint64),
+    ]
+
+
+class Apid(BaseStruct):
+
+    _fields_ = [
+        ('name', c.c_char * 16),
+        ('value', c.c_uint32),
+        ('pkt_tracker_start_idx', c.c_uint32),
+        ('pkts_reserved', c.c_uint32),
+        ('pkts_received', c.c_uint32),
+    ]
+
+
+class PacketTracker(BaseStruct):
+
+    _fields_ = [
+        ('obs_time', c.c_int64),
+        ('sequence_number', c.c_int32),
+        ('size', c.c_int32),
+        ('offset', c.c_int32),
+        ('fill_percent', c.c_int32),
+    ]
+
+
+def _make_packet(buf, size, offset):
+    """
+    Create a struct for a CCSDS packet. The struct size is dynamic so we need
+    a new class for each packet.
+
+    XXX: Many packets will have similar sizes so perhaps we should cache
+         packet impls based on their size.
+    """
+    # size minus the CCSDS primary header size
+    data_size = size - 6
+    class PktImpl(BaseStruct):  # noqa
+        _fields_ = [
+            ('version', c.c_uint8, 3),
+            ('type', c.c_uint8, 1),
+            ('secondary_header', c.c_uint8, 1),
+            ('apid', c.c_uint16, 11),
+            ('sequence_grouping', c.c_uint8, 2),
+            ('sequence_number', c.c_uint16, 14),
+            ('length_minus1', c.c_uint16),
+            ('data', c.c_uint8 * data_size)
+        ]
+    pkt = PktImpl.from_buffer(buf, offset)
+    assert pkt.length_minus1 + 1 == data_size
+    return pkt
+
+
+Packet = namedtuple('Packet', ('tracker', 'packet'))
+
+
+class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
+
+    def packets_for_apid(self, apid):
+        """
+        Return a generator that yields tuples of (tracker, packet) for the
+        given `Apid`.
+        """
+        return _packets_for_apid(self.buf, self.header, apid)
+
+    def packets(self):
+        """
+        Return a generator that yields `Packet`s for each apid in the
+        apid list.
+        """
+        for apid in self.apids:
+            for tracker, packet in self.packets_for_apid(apid):
+                yield Packet(tracker, packet)
+
+
+def _packets_for_apid(buf, header, apid):
+    t_off = header.pkt_tracker_offset + apid.pkt_tracker_start_idx * c.sizeof(PacketTracker)
+    for idx in range(apid.pkts_received):
+        tracker = PacketTracker.from_buffer(buf, t_off)
+        t_off += c.sizeof(PacketTracker)
+        if tracker.offset < 0:  # -1 == missing
+            continue
+
+        p_off = header.ap_storage_offset + tracker.offset
+        # packet = buf[p_off:p_off+tracker.size]
+        yield tracker, _make_packet(buf, tracker.size, p_off)
+
+
+def _read_apid_list(header, buf):
+    """
+    Return a generator that yields `Apid`s
+    """
+    start = header.apid_list_offset
+    end = header.num_apids * c.sizeof(Apid)
+    for offset in range(start, end, c.sizeof(Apid)):
+        yield Apid.from_buffer(buf, offset)
+
+
+def read_common_rdrs(sensor, filepath):
+    """
+    Return a generator that yields `CommonRdr` for each dataset provided by
+    `read_rdr_datasets`.
+    """
+    for buf in read_rdr_datasets(sensor, filepath):
+        header = StaticHeader.from_buffer(buf)
+        apids = _read_apid_list(header, buf)
+        yield CommonRdr(buf, header, list(apids))
+
+
+def read_rdr_datasets(sensor, filepath):
+    """
+    Return a generator that yields bytearrays for each RawApplicationPackets
+    dataset in numerical order.
+    """
+    sensor = sensor.upper()
+    fobj = H5File(filepath)
+    grp = fobj['/All_Data/{}-SCIENCE-RDR_All'.format(sensor)]
+
+    dsnames = [k for k in grp.keys() if k.startswith('RawApplicationPackets_')]
+    # compare ds names by index after '_'
+    cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup))  # noqa
+    for name in sorted(dsnames, cmp=cmp_names):
+        ds = grp[name]
+        yield np.getbuffer(np.array(ds))
+
+
+def sort_packets_by_obs_time(packets):
+    """
+    Sort `Packet`s in-place by the PacketTracker obs_time (IET).
+    """
+    return sorted(packets, key=lambda p: p.tracker.obs_time)
+
+
+def sort_packets_by_apid(packets, order=None):
+    """
+    Sort packets in-place by apid. An error will be raised if `order` is given
+    and packet has an apid not in the list.
+    """
+    if order:
+        return sorted(packets, key=lambda p: order.index(p.packet.apid))
+    else:
+        return sorted(packets, key=lambda p: p.packet.apid)
+
+
+def sort_packets_edos(rdr, packets):
+    """
+    Sort packets by timestamp and order the apids the same as EDOS NASA L0
+    files.
+    """
+    order = None
+    if rdr.header.sensor.lower() == 'viirs':
+        order = [826, 821] + range(0, 825)
+    return sort_packets_by_obs_time(sort_packets_by_apid(packets, order=order))
+
+
+def convert_to_nasa_l0(sensor, filename, skipfill=False, outfn=None):
+    """
+    Convert a JSPP RDR to a NASA L0 PDS file.
+    """
+    outfn = outfn or '{}.pds'.format(filename)
+    with open(outfn, 'wb') as fptr:
+        for idx, rdr in enumerate(read_common_rdrs(sensor, filename)):
+            packets = sort_packets_edos(rdr, rdr.packets())
+            for packet in packets:
+                if packet.tracker.fill_percent != 0:
+                    LOG.debug('fill: %s', packet.tracker)
+                    if skipfill:
+                        continue
+                fptr.write(packet.packet)
+    return outfn
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--output')
+    parser.add_argument('-f', '--skipfill', action='store_true')
+    parser.add_argument('sensor', choices=('viirs', 'atms', 'cris'))
+    parser.add_argument('rdr')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG)
+    convert_to_nasa_l0(args.sensor, args.rdr, outfn=args.output)