From e8b46f71e470f7bd202ae96c403224196f2ea912 Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Tue, 8 Dec 2015 15:53:05 +0000
Subject: [PATCH] Handle PacketTooShort in PacketStream
---
edosl0util/stream.py | 39 ++++++++++++++++++++++++++++++++++-----
setup.py | 2 +-
2 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/edosl0util/stream.py b/edosl0util/stream.py
index 746d821..6d9048c 100644
--- a/edosl0util/stream.py
+++ b/edosl0util/stream.py
@@ -36,7 +36,7 @@ class PacketTooShort(Error):
class NonConsecutiveSeqId(Error):
"""
- Non-consecutive sequence numbers encounterd for apid.
+ Non-consecutive sequence numbers encounterd for apid, i.e., packet gap.
"""
@@ -70,8 +70,8 @@ class BasicStream(object):
raise StopIteration()
if len(buf) != size:
raise PacketTooShort(
- 'expected to read {:d} bytes, got {:d}'
- .format(size, len(buf)))
+ 'expected to read {:d} bytes, got {:d} at offset {:d}'
+ .format(size, len(buf), self._offset), self.file)
self._offset += size
return buf
@@ -155,7 +155,20 @@ class Packet(object):
class PacketStream(object):
SEQID_NOTSET = -1
- def __init__(self, data_stream, fail_on_missing=False):
+ def __init__(self, data_stream, fail_on_missing=False, fail_on_tooshort=False):
+ """
+ :param data_stream: An interable of ``Tracker`` objects
+ :keyword fail_on_missing:
+ Raise a ``NonConsecutiveSeqId`` error when sequence id gaps are
+ found. If this is False sequence id gaps are ignored.
+ :keyword fail_on_tooshort:
+ Raise a ``PacketTooShort`` error when a packet is encountered for
+ which less bytes can be read than are expected according to the
+ packet headers. Most of the this would be due to a truncated file
+ with an incomplete packet at the end. If this is False a
+ ``StopIteration`` will be raised which will effectively truncate
+ the file.
+ """
self._stream = data_stream
self._seek_cache = deque()
self._first = None
@@ -165,6 +178,7 @@ class PacketStream(object):
'last_seqid': self.SEQID_NOTSET,
'num_missing': 0})
self._fail_on_missing = fail_on_missing
+ self._fail_on_tooshort = fail_on_tooshort
def __repr__(self):
filepath = getattr(self.file, 'name', None)
@@ -182,13 +196,24 @@ class PacketStream(object):
return self._stream.file
def push_back(self, packet):
+ """
+ Put a packet back such that it is the next file provided when
+ ``next`` is called.
+ """
self._seek_cache.append(packet)
def next(self):
# return any items on the seek cache before reading more
if len(self._seek_cache):
return self._seek_cache.popleft()
- h1, h2, data_size, offset, data = self._stream.next()
+ try:
+ h1, h2, data_size, offset, data = self._stream.next()
+ except PakcetTooShort as err:
+ if self._fail_on_tooshort:
+ raise
+ LOG.error('Packet too short, aborting stream: {:s}', err)
+ # The result of this is essentially a file truncation
+ raise StopIteration()
packet = Packet(h1, h2, data, data_size=data_size, offset=offset)
have_missing = self._update_info(packet)
if have_missing and self._fail_on_missing:
@@ -197,6 +222,10 @@ class PacketStream(object):
return packet
def _update_info(self, packet):
+ """
+ Handle gap detection and first/last. Returns whether any missing
+ packets were detected.
+ """
have_missing = False
apid = self._apid_info[packet.apid]
if packet.stamp:
diff --git a/setup.py b/setup.py
index ec91bb1..6a69058 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ setup(
author='Bruce Flynn',
author_email='brucef@ssec.wisc.edu',
description='Utilities for working with EDOS L0 PDS files',
- version='0.6.1',
+ version='0.6.2',
zip_safe=False,
packages=find_packages(),
pyver=True,
--
GitLab