diff --git a/edosl0util/stream.py b/edosl0util/stream.py index 746d82143c9afc6f3bfd5e0a115f9d6e7c6b5704..6d9048cc90eeb00495c16d5405693bd864d86d0f 100644 --- a/edosl0util/stream.py +++ b/edosl0util/stream.py @@ -36,7 +36,7 @@ class PacketTooShort(Error): class NonConsecutiveSeqId(Error): """ - Non-consecutive sequence numbers encounterd for apid. + Non-consecutive sequence numbers encounterd for apid, i.e., packet gap. """ @@ -70,8 +70,8 @@ class BasicStream(object): raise StopIteration() if len(buf) != size: raise PacketTooShort( - 'expected to read {:d} bytes, got {:d}' - .format(size, len(buf))) + 'expected to read {:d} bytes, got {:d} at offset {:d}' + .format(size, len(buf), self._offset), self.file) self._offset += size return buf @@ -155,7 +155,20 @@ class Packet(object): class PacketStream(object): SEQID_NOTSET = -1 - def __init__(self, data_stream, fail_on_missing=False): + def __init__(self, data_stream, fail_on_missing=False, fail_on_tooshort=False): + """ + :param data_stream: An interable of ``Tracker`` objects + :keyword fail_on_missing: + Raise a ``NonConsecutiveSeqId`` error when sequence id gaps are + found. If this is False sequence id gaps are ignored. + :keyword fail_on_tooshort: + Raise a ``PacketTooShort`` error when a packet is encountered for + which less bytes can be read than are expected according to the + packet headers. Most of the this would be due to a truncated file + with an incomplete packet at the end. If this is False a + ``StopIteration`` will be raised which will effectively truncate + the file. + """ self._stream = data_stream self._seek_cache = deque() self._first = None @@ -165,6 +178,7 @@ class PacketStream(object): 'last_seqid': self.SEQID_NOTSET, 'num_missing': 0}) self._fail_on_missing = fail_on_missing + self._fail_on_tooshort = fail_on_tooshort def __repr__(self): filepath = getattr(self.file, 'name', None) @@ -182,13 +196,24 @@ class PacketStream(object): return self._stream.file def push_back(self, packet): + """ + Put a packet back such that it is the next file provided when + ``next`` is called. + """ self._seek_cache.append(packet) def next(self): # return any items on the seek cache before reading more if len(self._seek_cache): return self._seek_cache.popleft() - h1, h2, data_size, offset, data = self._stream.next() + try: + h1, h2, data_size, offset, data = self._stream.next() + except PakcetTooShort as err: + if self._fail_on_tooshort: + raise + LOG.error('Packet too short, aborting stream: {:s}', err) + # The result of this is essentially a file truncation + raise StopIteration() packet = Packet(h1, h2, data, data_size=data_size, offset=offset) have_missing = self._update_info(packet) if have_missing and self._fail_on_missing: @@ -197,6 +222,10 @@ class PacketStream(object): return packet def _update_info(self, packet): + """ + Handle gap detection and first/last. Returns whether any missing + packets were detected. + """ have_missing = False apid = self._apid_info[packet.apid] if packet.stamp: diff --git a/setup.py b/setup.py index ec91bb1ebbd2b63ce9fca5a87ab8ba9972d56e02..6a69058ff4e2520685744aa4e5464e3ef7468d18 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( author='Bruce Flynn', author_email='brucef@ssec.wisc.edu', description='Utilities for working with EDOS L0 PDS files', - version='0.6.1', + version='0.6.2', zip_safe=False, packages=find_packages(), pyver=True,