diff --git a/edosl0util/rdrgen.py b/edosl0util/rdrgen.py
index b16fdcfad47a212c22d5e11fabc3900c0e08985f..cf527b41dc69e949094867b9c7761d2fe2f951ee 100644
--- a/edosl0util/rdrgen.py
+++ b/edosl0util/rdrgen.py
@@ -16,6 +16,17 @@ from edosl0util.jpssrdr import (
from edosl0util.stream import jpss_packet_stream
+class GetJpssPacketTime(object):
+ def __init__(self):
+ self._viirs_tracker = ViirsGroupedPacketTimeTracker()
+
+ def __call__(self, pkt):
+ if self._viirs_tracker.handles(pkt.apid):
+ return self._viirs_tracker.get_iet(pkt)
+ else:
+ return get_packet_iet(pkt)
+
+
class ViirsGroupedPacketTimeTracker(object):
grouped_apids = list(range(800, 824)) + [825]
@@ -23,21 +34,20 @@ class ViirsGroupedPacketTimeTracker(object):
def handles(cls, apid):
return apid in cls.grouped_apids
- def __init_(self):
- self.db = {}
+ def __init__(self):
+ self._db = {}
def get_iet(self, pkt):
if not self.handles(pkt.apid):
raise ValueError('APID {} not a VIIRS grouped packet type'.format(pkt.apid))
- viirs_iet = self.get_viirs_iet(pkt)
if pkt.is_first():
obs_iet = get_packet_iet(pkt)
- self.db[pkt.apid] = (obs_iet, pkt.seq_id)
+ self._db[pkt.apid] = (obs_iet, pkt.seqid)
return obs_iet
else:
- last_obs_iet, last_seq = self.db[pkt.apid]
- group_size = self.get_group_size(pkt.apid)
- if not self.check_sequence_number(pkt.seq_id, last_seq, group_size):
+ last_obs_iet, last_seq = self._db[pkt.apid]
+ group_size = ViirsScienceApidInfo.get_packets_per_scan(pkt.apid)
+ if not self.check_sequence_number(pkt.seqid, last_seq, group_size):
raise OrphanedViirsPacket(pkt)
if not self.check_packet_iet(self.get_viirs_iet(pkt), last_obs_iet):
raise OrphanedViirsPacket(pkt)
@@ -45,24 +55,18 @@ class ViirsGroupedPacketTimeTracker(object):
@staticmethod
def get_viirs_iet(pkt):
- idx = 20 if pkt.is_first() else 10
+ if pkt.is_standalone():
+ idx = 18
+ elif pkt.is_first():
+ idx = 20
+ else:
+ idx = 10
arr = np.frombuffer(pkt.bytes()[idx:idx+8], 'B')
days = arr[0:2].view('>u2')[0]
ms = arr[2:6].view('>u4')[0]
us = arr[6:8].view('>u2')[0]
return timecode_parts_to_iet(days, ms, us)
- @staticmethod
- def get_group_size(apid):
- i_band_apids = [813, 817, 818, 819, 820]
- cal_apid = 825
- if apid in i_band_apids:
- return 32
- elif apid == cal_apid:
- return 24
- else:
- return 16
-
@staticmethod
def check_sequence_number(nonfirst_seq_num, first_seq_num, group_size):
seq_limit = 2**14
@@ -92,19 +96,23 @@ class OrphanedViirsPacket(Exception):
def packets_to_rdrs(sat, pkt_files):
# FIXME: refactor!!!
rdr_pkt_files = {}
+ get_jpss_packet_time = GetJpssPacketTime()
for pkt_file in pkt_files:
with open(pkt_file) as file_obj:
stream = jpss_packet_stream(file_obj)
for pkt in stream:
rdr_type = get_rdr_type(pkt.apid)
- gran = calc_rdr_granule(sat, rdr_type, pkt)
+ pkt_iet = get_jpss_packet_time(pkt)
+ gran = calc_rdr_granule(sat, rdr_type, pkt_iet)
if (rdr_type, gran) not in rdr_pkt_files:
rdr_pkt_files[rdr_type, gran] = TemporaryFile()
rdr_pkt_files[rdr_type, gran].write(pkt.bytes())
+ get_jpss_packet_time = GetJpssPacketTime()
for rdr_pkt_file in rdr_pkt_files.values():
rdr_pkt_file.seek(0)
- pkts = jpss_packet_stream(rdr_pkt_file)
- pkts = sorted(pkts, key=(lambda p: (get_packet_iet(p), p.apid)))
+ pkts = list(jpss_packet_stream(rdr_pkt_file))
+ pkt_times = {p: get_jpss_packet_time(p) for p in pkts}
+ pkts.sort(key=(lambda p: (pkt_times[p], p.apid)))
blob = build_rdr_blob(sat, pkts)
write_rdr(sat, blob)
@@ -204,9 +212,10 @@ def set_h5_attrs(h5_obj, attrs):
def build_rdr_blob(sat, pkt_stream):
pkt_stream = iter(pkt_stream)
+ get_jpss_packet_time = GetJpssPacketTime()
first_pkt = next(pkt_stream) # FIXME: what if there are no packets?
rdr_type = get_rdr_type(first_pkt.apid)
- granule_iet = calc_rdr_granule(sat, rdr_type, first_pkt)
+ granule_iet = calc_rdr_granule(sat, rdr_type, get_jpss_packet_time(first_pkt))
granule_iet_end = granule_iet + rdr_type.gran_len
total_pkt_size = 0
@@ -225,7 +234,7 @@ def build_rdr_blob(sat, pkt_stream):
if pkt.apid not in apid_info:
raise ValueError(
'APID {} not expected for {}'.format(pkt.apid, rdr_type.short_name))
- pkt_iet = get_packet_iet(pkt)
+ pkt_iet = get_jpss_packet_time(pkt)
if not granule_iet <= pkt_iet < granule_iet_end:
raise ValueError('packet stream crosses granule boundary')
info = apid_info[pkt.apid]
@@ -283,6 +292,40 @@ def build_rdr_blob(sat, pkt_stream):
return buf
+class ViirsScienceApidInfo(object):
+ apids = list(x for x in range(800, 827) if x != 824)
+ names = ['M04', 'M05', 'M03', 'M02', 'M01', 'M06', 'M07', 'M09', 'M10',
+ 'M08', 'M11', 'M13', 'M12', 'I04', 'M16', 'M15', 'M14', 'I05',
+ 'I05', 'I01', 'I02', 'I03', 'DNB', 'DNB_MGS', 'DNB_LGS',
+ 'CAL', 'ENG']
+
+ @classmethod
+ def get_specs(cls):
+ return [ApidSpec(apid, cls.get_name(apid), cls.get_max_expected(apid))
+ for apid in cls.apids]
+
+ @classmethod
+ def get_name(cls, apid):
+ return cls.names[cls.apids.index(apid)]
+
+ @classmethod
+ def get_max_expected(cls, apid):
+ max_scans = 48
+ return max_scans * cls.get_packets_per_scan(apid)
+
+ @classmethod
+ def get_packets_per_scan(cls, apid):
+ name = cls.get_name(apid)
+ if name == 'ENG':
+ return 1
+ elif name == 'CAL':
+ return 24
+ elif name.startswith('M'):
+ return 17
+ else:
+ return 33
+
+
def get_cris_science_apids():
return ([ApidSpec(1289, 'EIGHT_S_SCI', 5), ApidSpec(1290, 'ENG', 1)]
+ get_cris_obs_apids())
@@ -345,6 +388,17 @@ rdr_type_spec = rdr_type_mgr.register_type
get_rdr_type = rdr_type_mgr.get_type_for_apid
+@rdr_type_spec
+class ViirsScienceRdrType(object):
+ product_id = 'RVIRS'
+ short_name = 'VIIRS-SCIENCE-RDR'
+ gran_len = 85350000
+ sensor = 'viirs'
+ type_id = 'SCIENCE'
+ document = '474-00448-02-08_JPSS-DD-Vol-II-Part-8_0200B.pdf'
+ apids = ViirsScienceApidInfo.get_specs()
+
+
@rdr_type_spec
class CrisScienceRdrType(object):
product_id = 'RCRIS'
@@ -401,9 +455,8 @@ def iet_to_datetime(iet):
return (iet_epoch + TimeDelta(iet * 1e-6, format='sec')).utc.datetime
-def calc_rdr_granule(sat, rdr_type, pkt):
- return calc_iet_granule(satellite_base_times[sat], rdr_type.gran_len,
- get_packet_iet(pkt))
+def calc_rdr_granule(sat, rdr_type, pkt_iet):
+ return calc_iet_granule(satellite_base_times[sat], rdr_type.gran_len, pkt_iet)
def calc_iet_granule(base_time, gran_len, iet):
@@ -435,7 +488,7 @@ def calc_percent_missing(common_rdr):
iet_epoch = Time('1958-01-01', scale='tai')
satellite_base_times = {'snpp': 1698019234000000}
platform_short_names = {'snpp': 'NPP'}
-instrument_short_names = {'cris': 'CrIS', None: 'SPACECRAFT'}
+instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', None: 'SPACECRAFT'}
default_origin = 'ssec'
default_domain = 'dev'
diff --git a/tests/test_rdrgen.py b/tests/test_rdrgen.py
index beb6fef13647110a7df7a7af481f01d6f0a7e1ed..2b9023c406d3d0ff216b0f8f9205a5385cd632b9 100644
--- a/tests/test_rdrgen.py
+++ b/tests/test_rdrgen.py
@@ -11,7 +11,45 @@ from edosl0util.stream import jpss_packet_stream
class TestViirsGroupedPacketTimeTracker(object):
- pass
+ def test_check_sequence_number(self):
+ group_size = 24
+ def run(nonfirst_seq_num, first_seq_num):
+ return m.ViirsGroupedPacketTimeTracker.check_sequence_number(
+ nonfirst_seq_num, first_seq_num, group_size)
+
+ first_seq = 4096
+ assert not run(4095, first_seq)
+ assert run(4097, first_seq)
+ assert run(4119, first_seq)
+ assert not run(4120, first_seq)
+
+ max_seq = 2**14
+ first_seq = max_seq - 16
+ assert not run(max_seq - 17, first_seq)
+ assert run(max_seq - 15, first_seq)
+ assert run(max_seq - 1, first_seq)
+ assert run(0, first_seq)
+ assert run(7, first_seq)
+ assert not run(8, first_seq)
+
+ def test_get_viirs_iet(self):
+ def run(pkt):
+ return m.iet_to_datetime(m.ViirsGroupedPacketTimeTracker.get_viirs_iet(pkt))
+
+ with open(self.l0_path) as l0_file:
+ stream = jpss_packet_stream(l0_file)
+ standalone_pkt = next(p for p in stream if p.is_standalone())
+ first_pkt = next(p for p in stream if p.is_first())
+ nonfirst_pkt = next(p for p in stream if p.is_continuing())
+
+ # expected values haven't been independently verified, just looked at
+ # to see that they at least make sense
+ assert run(standalone_pkt) == datetime(2017, 9, 27, 13, 54, 1, 727765)
+ assert run(first_pkt) == datetime(2017, 9, 27, 13, 54, 1, 746898)
+ assert run(nonfirst_pkt) == datetime(2017, 9, 27, 13, 54, 1, 748328)
+
+ l0_file = 'P1570826VIIRSSCIENCE6T17270135400001.PDS'
+ l0_path = os.path.join(os.path.dirname(__file__), l0_file)
def test_can_reproduce_rdr_from_class():