Handle PacketTooShort in PacketStream

edosl0util/stream.py

@@ -36,7 +36,7 @@ class PacketTooShort(Error):
 
 class NonConsecutiveSeqId(Error):
     """
-    Non-consecutive sequence numbers encounterd for apid.
+    Non-consecutive sequence numbers encounterd for apid, i.e., packet gap.
     """
 
 
@@ -70,8 +70,8 @@ class BasicStream(object):
             raise StopIteration()
         if len(buf) != size:
             raise PacketTooShort(
-                'expected to read {:d} bytes, got {:d}'
-                .format(size, len(buf)))
+                'expected to read {:d} bytes, got {:d} at offset {:d}'
+                .format(size, len(buf), self._offset), self.file)
         self._offset += size
         return buf
 
@@ -155,7 +155,20 @@ class Packet(object):
 class PacketStream(object):
     SEQID_NOTSET = -1
 
-    def __init__(self, data_stream, fail_on_missing=False):
+    def __init__(self, data_stream, fail_on_missing=False, fail_on_tooshort=False):
+        """
+        :param data_stream: An interable of ``Tracker`` objects
+        :keyword fail_on_missing:
+            Raise a ``NonConsecutiveSeqId`` error when sequence id gaps are
+            found. If this is False sequence id gaps are ignored.
+        :keyword fail_on_tooshort:
+            Raise a ``PacketTooShort`` error when a packet is encountered for
+            which less bytes can be read than are expected according to the
+            packet headers. Most of the this would be due to a truncated file
+            with an incomplete packet at the end. If this is False a
+            ``StopIteration`` will be raised which will effectively truncate
+            the file.
+        """
         self._stream = data_stream
         self._seek_cache = deque()
         self._first = None
@@ -165,6 +178,7 @@ class PacketStream(object):
                      'last_seqid': self.SEQID_NOTSET,
                      'num_missing': 0})
         self._fail_on_missing = fail_on_missing
+        self._fail_on_tooshort = fail_on_tooshort
 
     def __repr__(self):
         filepath = getattr(self.file, 'name', None)
@@ -182,13 +196,24 @@ class PacketStream(object):
         return self._stream.file
 
     def push_back(self, packet):
+        """
+        Put a packet back such that it is the next file provided when
+        ``next`` is called.
+        """
         self._seek_cache.append(packet)
 
     def next(self):
         # return any items on the seek cache before reading more
         if len(self._seek_cache):
             return self._seek_cache.popleft()
-        h1, h2, data_size, offset, data = self._stream.next()
+        try:
+            h1, h2, data_size, offset, data = self._stream.next()
+        except PakcetTooShort as err:
+            if self._fail_on_tooshort:
+                raise
+            LOG.error('Packet too short, aborting stream: {:s}', err)
+            # The result of this is essentially a file truncation
+            raise StopIteration()
         packet = Packet(h1, h2, data, data_size=data_size, offset=offset)
         have_missing = self._update_info(packet)
         if have_missing and self._fail_on_missing:
@@ -197,6 +222,10 @@ class PacketStream(object):
         return packet
 
     def _update_info(self, packet):
+        """
+        Handle gap detection and first/last. Returns whether any missing
+        packets were detected.
+        """
         have_missing = False
         apid = self._apid_info[packet.apid]
         if packet.stamp:

setup.py

@@ -5,7 +5,7 @@ setup(
     author='Bruce Flynn',
     author_email='brucef@ssec.wisc.edu',
     description='Utilities for working with EDOS L0 PDS files',
-    version='0.6.1',
+    version='0.6.2',
     zip_safe=False,
     packages=find_packages(),
     pyver=True,