diff --git a/edosl0util/cli/rdrmerge.py b/edosl0util/cli/rdrmerge.py index b91ab3d1e932d44deb066b610b693dcb35000e4b..575b788475a37f137f93277005dc29c02ca865d3 100644 --- a/edosl0util/cli/rdrmerge.py +++ b/edosl0util/cli/rdrmerge.py @@ -7,7 +7,7 @@ from os.path import basename, join from tempfile import mkdtemp from ..jpssrdr import atms_sci_to_l0, cris_sci_to_l0, spacecraft_to_l0, viirs_sci_to_l0 -from ..rdrgen import packets_to_rdrs +from ..rdrgen import build_rdr, filter_group_orphans, iter_pkts from ..stream import jpss_packet_stream from ..merge import merge from .util import configure_logging @@ -100,7 +100,8 @@ def merge_rdrs(inputs): with open(merged, 'wb') as fp: merge([jpss_packet_stream(open(p, 'rb')) for p in pds], fp) - rdrs = packets_to_rdrs(sat, [merged], aggr_type="full", output_dir=tmpdir) + packets = filter_group_orphans(iter_pkts([merged])) + rdrs = build_rdr(sat, packets, aggr_type="full", output_dir=tmpdir) assert len(rdrs) == 1, "Should have gotten a single RDR" rdr = rdrs[0] diff --git a/edosl0util/rdrgen.py b/edosl0util/rdrgen.py index 36018de584113f7dede2a4c82bdaee38e3214cb9..a9d2ac2101181006199ba8ecc39ccde70a12d784 100644 --- a/edosl0util/rdrgen.py +++ b/edosl0util/rdrgen.py @@ -28,42 +28,26 @@ def iter_pkts(l0_files): def filter_group_orphans(s): """ Filter out any packets in a group that do not fall withing the expected - sequence id series and expected packets per group. + sequence id series and expected packets per group. This really only + applies to VIIRS data since it's the only sensor that utilizes groups. This is useful for input RDR packet streams received from direct braodcast that may have sub-ideal packet sequences that RDR generation does not play well with. """ - seqnums = {} viirs_apids = set(ViirsScienceApidInfo.apids) - + viirs_tracker = ViirsGroupedPacketTimeTracker() for p in iter(s): - if p.apid in viirs_apids: - group_size = ViirsScienceApidInfo.get_packets_per_scan(p.apid) - # Not groupped - if group_size == 1: - yield p - continue - # Start of new group - if p.is_first(): - seqnums[p.apid] = p.seqid - yield p - continue - # If not in seqnums we must not have seen a first packet for this - # APID so it must be hanging, drop it. - if p.apid not in seqnums: - continue - # Packet does not have sequence number expected to be part of this - # group, must be an orphan, drop it. - # Ok to assume VIIRS here because it's the only one using groups. - if not ViirsGroupedPacketTimeTracker.check_sequence_number( - p.seqid, seqnums[p.apid], group_size - ): + # If it's a VIIRS grouped packet, check for orphans + if ( + p.apid in viirs_apids + and ViirsScienceApidInfo.get_packets_per_scan(p.apid) > 1 + ): + try: + viirs_tracker.get_iet(p) + except OrphanedViirsPacket: continue - yield p - - else: - yield p + yield p def packets_to_rdrs(sat, l0_files, **kwargs): @@ -767,6 +751,9 @@ class ViirsGroupedPacketTimeTracker(object): self._db[pkt.apid] = (obs_iet, pkt.seqid) return obs_iet else: + if pkt.apid not in self._db: + # have not seen a first packet yet + raise OrphanedViirsPacket(pkt) last_obs_iet, last_seq = self._db[pkt.apid] group_size = ViirsScienceApidInfo.get_packets_per_scan(pkt.apid) if not self.check_sequence_number(pkt.seqid, last_seq, group_size): diff --git a/tests/test_rdrgen.py b/tests/test_rdrgen.py index da92a0c05c14172622e8e346be0714c6a6495dc5..36a7d9d82ad1b9951b879e7f52e452cb2cb0253b 100644 --- a/tests/test_rdrgen.py +++ b/tests/test_rdrgen.py @@ -47,7 +47,7 @@ def test_can_reproduce_cris_rdr(tmpdir): def verify_rdr_reproduction(orig_file_name, tmp_dir, **build_rdr_opts): - orig_file = os.path.join(os.path.dirname(__file__), orig_file_name) + orig_file = os.path.join(os.path.dirname(__file__), 'fixtures', orig_file_name) new_file, = m.build_rdr('snpp', generate_rdr_packets(orig_file), output_dir=tmp_dir, **build_rdr_opts) new_file = os.path.join(tmp_dir, new_file) @@ -153,4 +153,4 @@ class TestViirsGroupedPacketTimeTracker(object): assert run(nonfirst_pkt) == datetime(2017, 9, 27, 13, 54, 1, 748328) l0_file = 'P1570826VIIRSSCIENCE6T17270135400001.PDS' - l0_path = os.path.join(os.path.dirname(__file__), l0_file) + l0_path = os.path.join(os.path.dirname(__file__), 'fixtures', l0_file)