From 4485a08152c6f1a185041252e3c180737b875bc8 Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Thu, 12 Dec 2019 16:09:00 -0600
Subject: [PATCH] rdrmerge filter packets before start-of-mission
---
edosl0util/cli/rdrmerge.py | 10 +++++++++-
edosl0util/rdrgen.py | 18 +++++++++++++++++-
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/edosl0util/cli/rdrmerge.py b/edosl0util/cli/rdrmerge.py
index 575b788..9f7bfbc 100644
--- a/edosl0util/cli/rdrmerge.py
+++ b/edosl0util/cli/rdrmerge.py
@@ -7,7 +7,7 @@ from os.path import basename, join
from tempfile import mkdtemp
from ..jpssrdr import atms_sci_to_l0, cris_sci_to_l0, spacecraft_to_l0, viirs_sci_to_l0
-from ..rdrgen import build_rdr, filter_group_orphans, iter_pkts
+from ..rdrgen import build_rdr, filter_group_orphans, filter_before, iter_pkts
from ..stream import jpss_packet_stream
from ..merge import merge
from .util import configure_logging
@@ -80,6 +80,12 @@ def extract_packets(sat, product, fpath, workdir="."):
return outputs
+start_of_mission = {
+ 'snpp': datetime(2011, 10, 28),
+ 'noaa20': datetime(2017, 11, 18),
+}
+
+
def merge_rdrs(inputs):
to_process = rdrs_to_process(inputs)
@@ -100,7 +106,9 @@ def merge_rdrs(inputs):
with open(merged, 'wb') as fp:
merge([jpss_packet_stream(open(p, 'rb')) for p in pds], fp)
+ # have to pre-filter "orphans" to prevent OrphanPacketError
packets = filter_group_orphans(iter_pkts([merged]))
+ packets = filter_before(packets, before=start_of_mission[sat])
rdrs = build_rdr(sat, packets, aggr_type="full", output_dir=tmpdir)
assert len(rdrs) == 1, "Should have gotten a single RDR"
rdr = rdrs[0]
diff --git a/edosl0util/rdrgen.py b/edosl0util/rdrgen.py
index 947a875..5561356 100644
--- a/edosl0util/rdrgen.py
+++ b/edosl0util/rdrgen.py
@@ -50,6 +50,22 @@ def filter_group_orphans(s):
yield p
+def filter_before(s, before):
+ """
+ Filter all packets that occur before ... before.
+ """
+ s = iter(s)
+ p = next(s)
+ done = object() # sentinal to stop iter
+ while p.stamp is None or p.stamp < before:
+ p = next(s, done)
+ if p is done:
+ return # return early, iter is done
+ yield p
+ for p in s:
+ yield p
+
+
def packets_to_rdrs(sat, l0_files, **kwargs):
return build_rdr(sat, iter_pkts(l0_files), **kwargs)
@@ -770,7 +786,7 @@ class ViirsGroupedPacketTimeTracker(object):
idx = 20
else:
idx = 10
- arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B")
+ arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B") # noqa
days = arr[0:2].view(">u2")[0]
ms = arr[2:6].view(">u4")[0]
us = arr[6:8].view(">u2")[0]
--
GitLab