rdrmerge filter packets before start-of-mission

edosl0util/cli/rdrmerge.py

@@ -7,7 +7,7 @@ from os.path import basename, join
 from tempfile import mkdtemp
 
 from ..jpssrdr import atms_sci_to_l0, cris_sci_to_l0, spacecraft_to_l0, viirs_sci_to_l0
-from ..rdrgen import build_rdr, filter_group_orphans, iter_pkts
+from ..rdrgen import build_rdr, filter_group_orphans, filter_before, iter_pkts
 from ..stream import jpss_packet_stream
 from ..merge import merge
 from .util import configure_logging
@@ -80,6 +80,12 @@ def extract_packets(sat, product, fpath, workdir="."):
     return outputs
 
 
+start_of_mission = {
+    'snpp': datetime(2011, 10, 28),
+    'noaa20': datetime(2017, 11, 18),
+}
+
+
 def merge_rdrs(inputs):
     to_process = rdrs_to_process(inputs)
 
@@ -100,7 +106,9 @@ def merge_rdrs(inputs):
             with open(merged, 'wb') as fp:
                 merge([jpss_packet_stream(open(p, 'rb')) for p in pds], fp)
 
+            # have to pre-filter "orphans" to prevent OrphanPacketError
             packets = filter_group_orphans(iter_pkts([merged]))
+            packets = filter_before(packets, before=start_of_mission[sat])
             rdrs = build_rdr(sat, packets, aggr_type="full", output_dir=tmpdir)
             assert len(rdrs) == 1, "Should have gotten a single RDR"
             rdr = rdrs[0]

edosl0util/rdrgen.py

@@ -50,6 +50,22 @@ def filter_group_orphans(s):
         yield p
 
 
+def filter_before(s, before):
+    """
+    Filter all packets that occur before ... before.
+    """
+    s = iter(s)
+    p = next(s)
+    done = object()  # sentinal to stop iter
+    while p.stamp is None or p.stamp < before:
+        p = next(s, done)
+        if p is done:
+            return  # return early, iter is done
+    yield p
+    for p in s:
+        yield p
+
+
 def packets_to_rdrs(sat, l0_files, **kwargs):
     return build_rdr(sat, iter_pkts(l0_files), **kwargs)
 
@@ -770,7 +786,7 @@ class ViirsGroupedPacketTimeTracker(object):
             idx = 20
         else:
             idx = 10
-        arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B")
+        arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B")  # noqa
         days = arr[0:2].view(">u2")[0]
         ms = arr[2:6].view(">u4")[0]
         us = arr[6:8].view(">u2")[0]