diff --git a/edosl0util/cli.py b/edosl0util/cli.py
index d8bceef6dc7ac833a70ed78311259b4fe62a87c1..2532f5015e57c01196fb8da15036b9ba892c446c 100644
--- a/edosl0util/cli.py
+++ b/edosl0util/cli.py
@@ -110,15 +110,23 @@ def cmd_merge():
def cmd_rdr2l0():
+ """
+ Extract CCSDS packets from a JPSS RDR. They are extracted in the order in
+ which they are listed in the APID list and in packet tracker order, i.e.,
+ Not in time/apid order. Files will be named using the input h5 name with
+ .science.pds and .diary[0,8,11].pds appended.
+ """
parser = _default_parser()
- parser.add_argument('-o', '--output')
- parser.add_argument('-f', '--skipfill', action='store_true')
- parser.add_argument('sensor', choices=('atms', 'cris'))
+ parser.description = cmd_rdr2l0.__doc__
+ parser.add_argument(
+ '-d', '--diary', action='store_true',
+ help='Write diary[0,8,11].pds if available')
+ parser.add_argument(
+ '-f', '--skipfill', action='store_true',
+ help=('Skip any packets that are marked in the packet tracker as '
+ 'containing fill'))
parser.add_argument('rdr')
args = parser.parse_args()
_configure_logging(args)
- output = args.output or args.rdr + '.pds'
- with io.open(output, 'wb') as fptr:
- for packet in jpssrdr.convert_to_nasa_l0(args.sensor, args.rdr):
- fptr.write(packet)
+ jpssrdr.write_rdr_datasets(args.rdr, args.diary)
diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py
index d8ef6a862c126e2d70f363d0754739d9c3ab2315..a217fae272350d01a365444e5c4998abe5ffcdda 100644
--- a/edosl0util/jpssrdr.py
+++ b/edosl0util/jpssrdr.py
@@ -9,8 +9,10 @@ Code for reading/writing/manipulating JPSS Common RDR files as documented in:
"""
__copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
+import os
import logging
import ctypes as c
+from datetime import datetime
from collections import namedtuple
import numpy as np
@@ -90,13 +92,17 @@ Packet = namedtuple('Packet', ('tracker', 'packet'))
class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
+ """
+ A single common rdr as found in a RawApplicationPackets_X dataset.
+ """
def packets_for_apid(self, apid):
"""
Return a generator that yields tuples of (tracker, packet) for the
given `Apid`.
"""
- return _packets_for_apid(self.buf, self.header, apid)
+ for tracker, packet in _packets_for_apid(self.buf, self.header, apid):
+ yield Packet(tracker, packet)
def packets(self):
"""
@@ -104,8 +110,8 @@ class CommonRdr(namedtuple('CommonRdr', ('buf', 'header', 'apids'))):
apid list.
"""
for apid in self.apids:
- for tracker, packet in self.packets_for_apid(apid):
- yield Packet(tracker, packet)
+ for packet in self.packets_for_apid(apid):
+ yield packet
def _packets_for_apid(buf, header, apid):
@@ -137,31 +143,54 @@ def _read_apid_list(header, buf):
offset += c.sizeof(Apid)
-def read_common_rdrs(sensor, filepath):
- """
- Generate CommonRdr-s for each dataset(granule) in `filelpath`
- """
- for buf in read_rdr_datasets(sensor, filepath):
- header = StaticHeader.from_buffer(buf)
- apids = _read_apid_list(header, buf)
- yield CommonRdr(buf, header, list(apids))
+def _sorted_packet_dataset_names(names):
+ names = [k for k in names if k.startswith('RawApplicationPackets_')]
+ cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup)) # noqa
+ return sorted(names, cmp=cmp_names)
-def read_rdr_datasets(sensor, filepath):
- """
- Generate byte arrays of granule RawApplicationPackets in granule number
- order.
- """
- sensor = sensor.upper()
+def _generate_packet_datasets(group):
+ dsnames = group.keys()
+ for name in _sorted_packet_dataset_names(dsnames):
+ ds = group[name]
+ yield np.array(ds)
+
+
+def _find_science_group(fobj):
+ for sensor in ['viirs', 'cris', 'atms']:
+ group = fobj.get('/All_Data/{}-SCIENCE-RDR_All'.format(sensor.upper()))
+ if group:
+ return group
+
+
+def _rdrs_for_packet_dataset(group):
+ if group:
+ for buf in _generate_packet_datasets(group):
+ header = StaticHeader.from_buffer(buf)
+ apids = _read_apid_list(header, buf)
+ yield CommonRdr(buf, header, list(apids))
+
+
+def rdr_datasets(filepath):
fobj = H5File(filepath)
- grp = fobj['/All_Data/{}-SCIENCE-RDR_All'.format(sensor)]
+ rdr = {}
+ group = _find_science_group(fobj)
+ rdr['science'] = _rdrs_for_packet_dataset(group)
+ group = fobj.get('/All_Data/SPACECRAFT-DIARY-RDR_All')
+ rdr['diary'] = _rdrs_for_packet_dataset(group)
+ return rdr
- dsnames = [k for k in grp.keys() if k.startswith('RawApplicationPackets_')]
- # compare ds names by index after '_'
- cmp_names = lambda *tup: cmp(*(int(x.split('_')[-1]) for x in tup)) # noqa
- for name in sorted(dsnames, cmp=cmp_names):
- ds = grp[name]
- yield np.array(ds)
+
+def rdr_ccsds_packets(rdr, skipfill=False):
+ """
+ Get all CCSDS packets from an `CommonRdr` in packet tracker order.
+ """
+ for packet in rdr.packets():
+ if packet.tracker.fill_percent != 0:
+ LOG.debug('fill: %s', packet.tracker)
+ if skipfill:
+ continue
+ yield packet.packet
def sort_packets_by_obs_time(packets):
@@ -182,17 +211,25 @@ def sort_packets_by_apid(packets, order=None):
return sorted(packets, key=lambda p: p.packet.apid)
-def convert_to_nasa_l0(sensor, filename, skipfill=False):
- """
- Convert a JSPP RDR to a NASA L0 PDS file.
- """
- for idx, rdr in enumerate(read_common_rdrs(sensor, filename)):
- LOG.debug(rdr.header)
- for apid in rdr.apids:
- LOG.debug(apid)
- for packet in rdr.packets():
- if packet.tracker.fill_percent != 0:
- LOG.debug('fill: %s', packet.tracker)
- if skipfill:
+def write_rdr_datasets(filepath, diary=True, skipfill=False):
+ def write_packets(pkts, dest):
+ with open(dest, 'wb') as fptr:
+ for pkt in pkts:
+ if pkt.tracker.fill_percent != 0 and skipfill:
continue
- yield packet.packet
+ fptr.write(pkt.packet)
+ rdrname = os.path.basename(filepath)
+ rdrs = rdr_datasets(filepath)
+ for idx, rdr in enumerate(rdrs['science']):
+ fname = '{}.science.pds'.format(rdrname)
+ hdr = rdr.header
+ LOG.debug('writing gran %d %s-%s-%s', idx, hdr.satellite, hdr.sensor, hdr.type_id)
+ write_packets(rdr.packets(), fname)
+
+ if diary:
+ for idx, rdr in enumerate(rdrs['diary']):
+ for apid in rdr.apids:
+ fname = '{}.diary{}.pds'.format(rdrname, apid.value)
+ hdr = rdr.header
+ LOG.debug('writing gran %d %s-%s-%s %d', idx, hdr.satellite, hdr.sensor, hdr.type_id, apid.value)
+ write_packets(rdr.packets_for_apid(apid), fname)