diff --git a/edosl0util/cli/rdrmerge.py b/edosl0util/cli/rdrmerge.py index 575b788475a37f137f93277005dc29c02ca865d3..9f7bfbc2eaa8322bdeee2503d4c1043f24c4275c 100644 --- a/edosl0util/cli/rdrmerge.py +++ b/edosl0util/cli/rdrmerge.py @@ -7,7 +7,7 @@ from os.path import basename, join from tempfile import mkdtemp from ..jpssrdr import atms_sci_to_l0, cris_sci_to_l0, spacecraft_to_l0, viirs_sci_to_l0 -from ..rdrgen import build_rdr, filter_group_orphans, iter_pkts +from ..rdrgen import build_rdr, filter_group_orphans, filter_before, iter_pkts from ..stream import jpss_packet_stream from ..merge import merge from .util import configure_logging @@ -80,6 +80,12 @@ def extract_packets(sat, product, fpath, workdir="."): return outputs +start_of_mission = { + 'snpp': datetime(2011, 10, 28), + 'noaa20': datetime(2017, 11, 18), +} + + def merge_rdrs(inputs): to_process = rdrs_to_process(inputs) @@ -100,7 +106,9 @@ def merge_rdrs(inputs): with open(merged, 'wb') as fp: merge([jpss_packet_stream(open(p, 'rb')) for p in pds], fp) + # have to pre-filter "orphans" to prevent OrphanPacketError packets = filter_group_orphans(iter_pkts([merged])) + packets = filter_before(packets, before=start_of_mission[sat]) rdrs = build_rdr(sat, packets, aggr_type="full", output_dir=tmpdir) assert len(rdrs) == 1, "Should have gotten a single RDR" rdr = rdrs[0] diff --git a/edosl0util/rdrgen.py b/edosl0util/rdrgen.py index 947a875fcb4500dd8399d5869af18c56e980fc17..55613568e8b56497349f7bf2597f591ecffd1489 100644 --- a/edosl0util/rdrgen.py +++ b/edosl0util/rdrgen.py @@ -50,6 +50,22 @@ def filter_group_orphans(s): yield p +def filter_before(s, before): + """ + Filter all packets that occur before ... before. + """ + s = iter(s) + p = next(s) + done = object() # sentinal to stop iter + while p.stamp is None or p.stamp < before: + p = next(s, done) + if p is done: + return # return early, iter is done + yield p + for p in s: + yield p + + def packets_to_rdrs(sat, l0_files, **kwargs): return build_rdr(sat, iter_pkts(l0_files), **kwargs) @@ -770,7 +786,7 @@ class ViirsGroupedPacketTimeTracker(object): idx = 20 else: idx = 10 - arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B") + arr = np.frombuffer(pkt.bytes()[idx : idx + 8], "B") # noqa days = arr[0:2].view(">u2")[0] ms = arr[2:6].view(">u4")[0] us = arr[6:8].view(">u2")[0]