From e626c21be1ab70f7e15cfaf25a97642cd9ea7a8e Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Sat, 26 Sep 2015 15:26:08 +0000
Subject: [PATCH] Add jpssrdr module
---
edosl0util/jpssrdr.py | 222 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 222 insertions(+)
create mode 100644 edosl0util/jpssrdr.py
diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py
new file mode 100644
index 0000000..0abed0d
--- /dev/null
+++ b/edosl0util/jpssrdr.py
@@ -0,0 +1,222 @@
+"""
+Code for reading/writing/manipulating JPSS Common RDR files as documented in:
+
+ Joint Polar Satellite System (JPSS)
+ Common Data Format Control Book - External (CDFCB-X)
+ Volume II - RDR Formats
+
+ http://jointmission.gsfc.nasa.gov/sciencedocs/2015-06/474-00001-02_JPSS-CDFCB-X-Vol-II_0123B.pdf
+"""
+import logging
+import ctypes as c
+from collections import namedtuple
+
+import numpy as np
+from h5py import File as H5File
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseStruct(c.BigEndianStructure):
+ _pack_ = 1
+ _fields_ = []
+
+ def __repr__(self):
+ attrs = ' '.join('%s=%s' % (f[0], getattr(self, f[0])) for f in self._fields_)
+ return '<{} {}>'.format(self.__class__.__name__, attrs)
+
+
+class StaticHeader(BaseStruct):
+
+ _fields_ = [
+ ('satellite', c.c_char * 4),
+ ('sensor', c.c_char * 16),
+ ('type_id', c.c_char * 16),
+ ('num_apids', c.c_uint32),
+ ('apid_list_offset', c.c_uint32),
+ ('pkt_tracker_offset', c.c_uint32),
+ ('ap_storage_offset', c.c_uint32),
+ ('next_pkt_pos', c.c_uint32),
+ ('start_boundry', c.c_uint64),
+ ('end_boundry', c.c_uint64),
+ ]
+
+
+class Apid(BaseStruct):
+
+ _fields_ = [
+ ('name', c.c_char * 16),
+ ('value', c.c_uint32),
+ ('pkt_tracker_start_idx', c.c_uint32),
+ ('pkts_reserved', c.c_uint32),
+ ('pkts_received', c.c_uint32),
+ ]
+
+
+class PacketTracker(BaseStruct):
+
+ _fields_ = [
+ ('obs_time', c.c_int64),
+ ('sequence_number', c.c_int32),
+ ('size', c.c_int32),
+ ('offset', c.c_int32),
+ ('fill_percent', c.c_int32),
+ ]
+
+
+def _make_packet(buf, size, offset):
+ """
+ Create a struct for a CCSDS packet. The struct size is dynamic so we need
+ a new class for each packet.
+
+ XXX: Many packets will have similar sizes so perhaps we should cache
+ packet impls based on their size.
+ """
+ # size minus the CCSDS primary header size
+ data_size = size - 6
+ class PktImpl(BaseStruct): # noqa
+ _fields_ = [
+ ('version', c.c_uint8, 3),
+ ('type', c.c_uint8, 1),
+ ('secondary_header', c.c_uint8, 1),
+ ('apid', c.c_uint16, 11),
+ ('sequence_grouping', c.c_uint8, 2),
+ ('sequence_number', c.c_uint16, 14),
+ ('length_minus1', c.c_uint16),
+ ('data', c.c_uint8 * data_size)
+ ]
+ pkt = PktImpl.from_buffer(buf, offset)
+ assert pkt.length_minus1 + 1 == data_size
+ return pkt
+
+
+Packet = namedtuple('Packet', ('tracker', 'packet'))
+
+
+class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
+
+ def packets_for_apid(self, apid):
+ """
+ Return a generator that yields tuples of (tracker, packet) for the
+ given `Apid`.
+ """
+ return _packets_for_apid(self.buf, self.header, apid)
+
+ def packets(self):
+ """
+ Return a generator that yields `Packet`s for each apid in the
+ apid list.
+ """
+ for apid in self.apids:
+ for tracker, packet in self.packets_for_apid(apid):
+ yield Packet(tracker, packet)
+
+
+def _packets_for_apid(buf, header, apid):
+ t_off = header.pkt_tracker_offset + apid.pkt_tracker_start_idx * c.sizeof(PacketTracker)
+ for idx in range(apid.pkts_received):
+ tracker = PacketTracker.from_buffer(buf, t_off)
+ t_off += c.sizeof(PacketTracker)
+ if tracker.offset < 0: # -1 == missing
+ continue
+
+ p_off = header.ap_storage_offset + tracker.offset
+ # packet = buf[p_off:p_off+tracker.size]
+ yield tracker, _make_packet(buf, tracker.size, p_off)
+
+
+def _read_apid_list(header, buf):
+ """
+ Return a generator that yields `Apid`s
+ """
+ start = header.apid_list_offset
+ end = header.num_apids * c.sizeof(Apid)
+ for offset in range(start, end, c.sizeof(Apid)):
+ yield Apid.from_buffer(buf, offset)
+
+
+def read_common_rdrs(sensor, filepath):
+ """
+ Return a generator that yields `CommonRdr` for each dataset provided by
+ `read_rdr_datasets`.
+ """
+ for buf in read_rdr_datasets(sensor, filepath):
+ header = StaticHeader.from_buffer(buf)
+ apids = _read_apid_list(header, buf)
+ yield CommonRdr(buf, header, list(apids))
+
+
+def read_rdr_datasets(sensor, filepath):
+ """
+ Return a generator that yields bytearrays for each RawApplicationPackets
+ dataset in numerical order.
+ """
+ sensor = sensor.upper()
+ fobj = H5File(filepath)
+ grp = fobj['/All_Data/{}-SCIENCE-RDR_All'.format(sensor)]
+
+ dsnames = [k for k in grp.keys() if k.startswith('RawApplicationPackets_')]
+ # compare ds names by index after '_'
+ cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup)) # noqa
+ for name in sorted(dsnames, cmp=cmp_names):
+ ds = grp[name]
+ yield np.getbuffer(np.array(ds))
+
+
+def sort_packets_by_obs_time(packets):
+ """
+ Sort `Packet`s in-place by the PacketTracker obs_time (IET).
+ """
+ return sorted(packets, key=lambda p: p.tracker.obs_time)
+
+
+def sort_packets_by_apid(packets, order=None):
+ """
+ Sort packets in-place by apid. An error will be raised if `order` is given
+ and packet has an apid not in the list.
+ """
+ if order:
+ return sorted(packets, key=lambda p: order.index(p.packet.apid))
+ else:
+ return sorted(packets, key=lambda p: p.packet.apid)
+
+
+def sort_packets_edos(rdr, packets):
+ """
+ Sort packets by timestamp and order the apids the same as EDOS NASA L0
+ files.
+ """
+ order = None
+ if rdr.header.sensor.lower() == 'viirs':
+ order = [826, 821] + range(0, 825)
+ return sort_packets_by_obs_time(sort_packets_by_apid(packets, order=order))
+
+
+def convert_to_nasa_l0(sensor, filename, skipfill=False, outfn=None):
+ """
+ Convert a JSPP RDR to a NASA L0 PDS file.
+ """
+ outfn = outfn or '{}.pds'.format(filename)
+ with open(outfn, 'wb') as fptr:
+ for idx, rdr in enumerate(read_common_rdrs(sensor, filename)):
+ packets = sort_packets_edos(rdr, rdr.packets())
+ for packet in packets:
+ if packet.tracker.fill_percent != 0:
+ LOG.debug('fill: %s', packet.tracker)
+ if skipfill:
+ continue
+ fptr.write(packet.packet)
+ return outfn
+
+
+if __name__ == '__main__':
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-o', '--output')
+ parser.add_argument('-f', '--skipfill', action='store_true')
+ parser.add_argument('sensor', choices=('viirs', 'atms', 'cris'))
+ parser.add_argument('rdr')
+ args = parser.parse_args()
+
+ logging.basicConfig(level=logging.DEBUG)
+ convert_to_nasa_l0(args.sensor, args.rdr, outfn=args.output)
--
GitLab