Get VIIRS science RDR gen working

edosl0util/rdrgen.py

@@ -16,6 +16,17 @@ from edosl0util.jpssrdr import (
 from edosl0util.stream import jpss_packet_stream
 
 
+class GetJpssPacketTime(object):
+    def __init__(self):
+        self._viirs_tracker = ViirsGroupedPacketTimeTracker()
+
+    def __call__(self, pkt):
+        if self._viirs_tracker.handles(pkt.apid):
+            return self._viirs_tracker.get_iet(pkt)
+        else:
+            return get_packet_iet(pkt)
+
+
 class ViirsGroupedPacketTimeTracker(object):
     grouped_apids = list(range(800, 824)) + [825]
 
@@ -23,21 +34,20 @@ class ViirsGroupedPacketTimeTracker(object):
     def handles(cls, apid):
         return apid in cls.grouped_apids
 
-    def __init_(self):
-        self.db = {}
+    def __init__(self):
+        self._db = {}
 
     def get_iet(self, pkt):
         if not self.handles(pkt.apid):
             raise ValueError('APID {} not a VIIRS grouped packet type'.format(pkt.apid))
-        viirs_iet = self.get_viirs_iet(pkt)
         if pkt.is_first():
             obs_iet = get_packet_iet(pkt)
-            self.db[pkt.apid] = (obs_iet, pkt.seq_id)
+            self._db[pkt.apid] = (obs_iet, pkt.seqid)
             return obs_iet
         else:
-            last_obs_iet, last_seq = self.db[pkt.apid]
-            group_size = self.get_group_size(pkt.apid)
-            if not self.check_sequence_number(pkt.seq_id, last_seq, group_size):
+            last_obs_iet, last_seq = self._db[pkt.apid]
+            group_size = ViirsScienceApidInfo.get_packets_per_scan(pkt.apid)
+            if not self.check_sequence_number(pkt.seqid, last_seq, group_size):
                 raise OrphanedViirsPacket(pkt)
             if not self.check_packet_iet(self.get_viirs_iet(pkt), last_obs_iet):
                 raise OrphanedViirsPacket(pkt)
@@ -45,24 +55,18 @@ class ViirsGroupedPacketTimeTracker(object):
 
     @staticmethod
     def get_viirs_iet(pkt):
-        idx = 20 if pkt.is_first() else 10
+        if pkt.is_standalone():
+            idx = 18
+        elif pkt.is_first():
+            idx = 20
+        else:
+            idx = 10
         arr = np.frombuffer(pkt.bytes()[idx:idx+8], 'B')
         days = arr[0:2].view('>u2')[0]
         ms = arr[2:6].view('>u4')[0]
         us = arr[6:8].view('>u2')[0]
         return timecode_parts_to_iet(days, ms, us)
 
-    @staticmethod
-    def get_group_size(apid):
-        i_band_apids = [813, 817, 818, 819, 820]
-        cal_apid = 825
-        if apid in i_band_apids:
-            return 32
-        elif apid == cal_apid:
-            return 24
-        else:
-            return 16
-
     @staticmethod
     def check_sequence_number(nonfirst_seq_num, first_seq_num, group_size):
         seq_limit = 2**14
@@ -92,19 +96,23 @@ class OrphanedViirsPacket(Exception):
 def packets_to_rdrs(sat, pkt_files):
     # FIXME: refactor!!!
     rdr_pkt_files = {}
+    get_jpss_packet_time = GetJpssPacketTime()
     for pkt_file in pkt_files:
         with open(pkt_file) as file_obj:
             stream = jpss_packet_stream(file_obj)
             for pkt in stream:
                 rdr_type = get_rdr_type(pkt.apid)
-                gran = calc_rdr_granule(sat, rdr_type, pkt)
+                pkt_iet = get_jpss_packet_time(pkt)
+                gran = calc_rdr_granule(sat, rdr_type, pkt_iet)
                 if (rdr_type, gran) not in rdr_pkt_files:
                     rdr_pkt_files[rdr_type, gran] = TemporaryFile()
                 rdr_pkt_files[rdr_type, gran].write(pkt.bytes())
+    get_jpss_packet_time = GetJpssPacketTime()
     for rdr_pkt_file in rdr_pkt_files.values():
         rdr_pkt_file.seek(0)
-        pkts = jpss_packet_stream(rdr_pkt_file)
-        pkts = sorted(pkts, key=(lambda p: (get_packet_iet(p), p.apid)))
+        pkts = list(jpss_packet_stream(rdr_pkt_file))
+        pkt_times = {p: get_jpss_packet_time(p) for p in pkts}
+        pkts.sort(key=(lambda p: (pkt_times[p], p.apid)))
         blob = build_rdr_blob(sat, pkts)
         write_rdr(sat, blob)
 
@@ -204,9 +212,10 @@ def set_h5_attrs(h5_obj, attrs):
 
 def build_rdr_blob(sat, pkt_stream):
     pkt_stream = iter(pkt_stream)
+    get_jpss_packet_time = GetJpssPacketTime()
     first_pkt = next(pkt_stream)  # FIXME: what if there are no packets?
     rdr_type = get_rdr_type(first_pkt.apid)
-    granule_iet = calc_rdr_granule(sat, rdr_type, first_pkt)
+    granule_iet = calc_rdr_granule(sat, rdr_type, get_jpss_packet_time(first_pkt))
     granule_iet_end = granule_iet + rdr_type.gran_len
 
     total_pkt_size = 0
@@ -225,7 +234,7 @@ def build_rdr_blob(sat, pkt_stream):
         if pkt.apid not in apid_info:
             raise ValueError(
                 'APID {} not expected for {}'.format(pkt.apid, rdr_type.short_name))
-        pkt_iet = get_packet_iet(pkt)
+        pkt_iet = get_jpss_packet_time(pkt)
         if not granule_iet <= pkt_iet < granule_iet_end:
             raise ValueError('packet stream crosses granule boundary')
         info = apid_info[pkt.apid]
@@ -283,6 +292,40 @@ def build_rdr_blob(sat, pkt_stream):
     return buf
 
 
+class ViirsScienceApidInfo(object):
+    apids = list(x for x in range(800, 827) if x != 824)
+    names = ['M04', 'M05', 'M03', 'M02', 'M01', 'M06', 'M07', 'M09', 'M10',
+             'M08', 'M11', 'M13', 'M12', 'I04', 'M16', 'M15', 'M14', 'I05',
+             'I05', 'I01', 'I02', 'I03', 'DNB', 'DNB_MGS', 'DNB_LGS',
+             'CAL', 'ENG']
+
+    @classmethod
+    def get_specs(cls):
+        return [ApidSpec(apid, cls.get_name(apid), cls.get_max_expected(apid))
+                for apid in cls.apids]
+
+    @classmethod
+    def get_name(cls, apid):
+        return cls.names[cls.apids.index(apid)]
+
+    @classmethod
+    def get_max_expected(cls, apid):
+        max_scans = 48
+        return max_scans * cls.get_packets_per_scan(apid)
+
+    @classmethod
+    def get_packets_per_scan(cls, apid):
+        name = cls.get_name(apid)
+        if name == 'ENG':
+            return 1
+        elif name == 'CAL':
+            return 24
+        elif name.startswith('M'):
+            return 17
+        else:
+            return 33
+
+
 def get_cris_science_apids():
     return ([ApidSpec(1289, 'EIGHT_S_SCI', 5), ApidSpec(1290, 'ENG', 1)]
             + get_cris_obs_apids())
@@ -345,6 +388,17 @@ rdr_type_spec = rdr_type_mgr.register_type
 get_rdr_type = rdr_type_mgr.get_type_for_apid
 
 
+@rdr_type_spec
+class ViirsScienceRdrType(object):
+    product_id = 'RVIRS'
+    short_name = 'VIIRS-SCIENCE-RDR'
+    gran_len = 85350000
+    sensor = 'viirs'
+    type_id = 'SCIENCE'
+    document = '474-00448-02-08_JPSS-DD-Vol-II-Part-8_0200B.pdf'
+    apids = ViirsScienceApidInfo.get_specs()
+
+
 @rdr_type_spec
 class CrisScienceRdrType(object):
     product_id = 'RCRIS'
@@ -401,9 +455,8 @@ def iet_to_datetime(iet):
     return (iet_epoch + TimeDelta(iet * 1e-6, format='sec')).utc.datetime
 
 
-def calc_rdr_granule(sat, rdr_type, pkt):
-    return calc_iet_granule(satellite_base_times[sat], rdr_type.gran_len,
-                            get_packet_iet(pkt))
+def calc_rdr_granule(sat, rdr_type, pkt_iet):
+    return calc_iet_granule(satellite_base_times[sat], rdr_type.gran_len, pkt_iet)
 
 
 def calc_iet_granule(base_time, gran_len, iet):
@@ -435,7 +488,7 @@ def calc_percent_missing(common_rdr):
 iet_epoch = Time('1958-01-01', scale='tai')
 satellite_base_times = {'snpp': 1698019234000000}
 platform_short_names = {'snpp': 'NPP'}
-instrument_short_names = {'cris': 'CrIS', None: 'SPACECRAFT'}
+instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', None: 'SPACECRAFT'}
 default_origin = 'ssec'
 default_domain = 'dev'
 

tests/test_rdrgen.py

@@ -11,7 +11,45 @@ from edosl0util.stream import jpss_packet_stream
 
 
 class TestViirsGroupedPacketTimeTracker(object):
-    pass
+    def test_check_sequence_number(self):
+        group_size = 24
+        def run(nonfirst_seq_num, first_seq_num):
+            return m.ViirsGroupedPacketTimeTracker.check_sequence_number(
+                nonfirst_seq_num, first_seq_num, group_size)
+
+        first_seq = 4096
+        assert not run(4095, first_seq)
+        assert run(4097, first_seq)
+        assert run(4119, first_seq)
+        assert not run(4120, first_seq)
+
+        max_seq = 2**14
+        first_seq = max_seq - 16
+        assert not run(max_seq - 17, first_seq)
+        assert run(max_seq - 15, first_seq)
+        assert run(max_seq - 1, first_seq)
+        assert run(0, first_seq)
+        assert run(7, first_seq)
+        assert not run(8, first_seq)
+
+    def test_get_viirs_iet(self):
+        def run(pkt):
+            return m.iet_to_datetime(m.ViirsGroupedPacketTimeTracker.get_viirs_iet(pkt))
+
+        with open(self.l0_path) as l0_file:
+            stream = jpss_packet_stream(l0_file)
+            standalone_pkt = next(p for p in stream if p.is_standalone())
+            first_pkt = next(p for p in stream if p.is_first())
+            nonfirst_pkt = next(p for p in stream if p.is_continuing())
+
+        # expected values haven't been independently verified, just looked at
+        # to see that they at least make sense
+        assert run(standalone_pkt) == datetime(2017, 9, 27, 13, 54, 1, 727765)
+        assert run(first_pkt) == datetime(2017, 9, 27, 13, 54, 1, 746898)
+        assert run(nonfirst_pkt) == datetime(2017, 9, 27, 13, 54, 1, 748328)
+
+    l0_file = 'P1570826VIIRSSCIENCE6T17270135400001.PDS'
+    l0_path = os.path.join(os.path.dirname(__file__), l0_file)
 
 
 def test_can_reproduce_rdr_from_class():