diff --git a/edosl0util/compat.py b/edosl0util/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb1ffe72bd12f5efb2ef1622332b3913741e6192
--- /dev/null
+++ b/edosl0util/compat.py
@@ -0,0 +1,5 @@
+str_types = (str,)
+try:
+ str_types += (unicode,)
+except:
+ pass
diff --git a/edosl0util/rdrgen.py b/edosl0util/rdrgen.py
index 58de617761d0a06a4ad97b0311c79b7ca95ebc53..1275472bf5284cdc9e56ac90c9237a0a4ddeac62 100644
--- a/edosl0util/rdrgen.py
+++ b/edosl0util/rdrgen.py
@@ -10,6 +10,7 @@ import h5py
import numpy as np
import edosl0util
+from edosl0util import compat
from edosl0util.jpssrdr import (
StaticHeader, Apid as ApidListItem, PacketTracker, decode_rdr_blob)
from edosl0util.stream import jpss_packet_stream
@@ -19,7 +20,7 @@ from edosl0util.timecode import cds_to_iet, iet_to_dt
def packets_to_rdrs(sat, l0_files, **kwargs):
def iter_pkts(l0_files):
for l0_file in l0_files:
- with open(l0_file) as l0_file_obj:
+ with open(l0_file, 'rb') as l0_file_obj:
for pkt in jpss_packet_stream(l0_file_obj):
yield pkt
@@ -157,7 +158,7 @@ class BinnedTemporaryFileManager(object):
if file_obj:
file_obj.close()
file_path = self._file_paths.pop(bin_key, '/dev/null')
- file_obj = open(file_path)
+ file_obj = open(file_path, 'rb')
try:
yield file_obj
finally:
@@ -272,6 +273,7 @@ class RdrWriter(object):
@staticmethod
def _set_h5_attrs(h5_obj, attrs):
+ attrs = _encodeall(attrs)
# for some reason all the H5 attributes are in 2-D arrays in IDPS files
for name, value in attrs.items():
if isinstance(value, list):
@@ -300,6 +302,22 @@ class RdrWriter(object):
_h5_regionref_dtype = h5py.special_dtype(ref=h5py.RegionReference)
+def _encodeall(v):
+ """
+ Recurse v encoding stringy values. dict keys are not encoded.
+ """
+ if isinstance(v, dict):
+ dtmp = {}
+ for k, v in v.items():
+ dtmp[k] = _encodeall(v)
+ return dtmp
+ if isinstance(v, list):
+ return [_encodeall(x) for x in v]
+ if isinstance(v, compat.str_types):
+ return v.encode()
+ return v
+
+
def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
get_jpss_packet_time = GetJpssPacketTime()
granule_iet_end = granule_iet + rdr_type.gran_len
@@ -310,8 +328,9 @@ def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
all_pkts = []
for apid in rdr_type.apids:
apid_info[apid.num] = {
- 'name': apid.name,
- 'pkts_reserved': apid.max_expected, 'pkts_received': 0,
+ 'name': apid.name.encode(),
+ 'pkts_reserved': apid.max_expected,
+ 'pkts_received': 0,
'first_tracker_index': total_trackers,
'pkt_info': [{} for _ in range(apid.max_expected)]}
total_trackers += apid.max_expected
@@ -340,9 +359,9 @@ def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
buf = np.zeros([buf_size], np.uint8) # zeros needed to null-pad strings
header = StaticHeader.from_buffer(buf)
- header.satellite = platform_short_names[sat]
- header.sensor = instrument_short_names[rdr_type.sensor]
- header.type_id = rdr_type.type_id
+ header.satellite = platform_short_names[sat].encode()
+ header.sensor = instrument_short_names[rdr_type.sensor].encode()
+ header.type_id = rdr_type.type_id.encode()
header.num_apids = len(apid_info)
header.apid_list_offset = apid_list_offset
header.pkt_tracker_offset = pkt_tracker_offset
@@ -625,7 +644,7 @@ def make_rdr_filename(rdr_types, sat, aggr_begin, aggr_end, orbit_num, creation_
origin = origin[:-1] + ('c' if compressed else 'u')
def format_time(t):
- return t.strftime('%H%M%S') + str(t.microsecond / 100000)
+ return t.strftime('%H%M%S') + str(t.microsecond // 100000)
return '{p}_{s}_d{d:%Y%m%d}_t{b}_e{e}_b{n:05d}_c{c:%Y%m%d%H%M%S%f}_{o}_{m}.h5'.format(
p=prod_ids, s=sat, d=aggr_begin, b=format_time(aggr_begin),
@@ -681,5 +700,4 @@ def get_overlapping_granules(sat, gran_len, start_iet, stop_iet):
satellite_base_times = {'snpp': 1698019234000000}
platform_short_names = {'snpp': 'NPP'}
-instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', 'atms': 'ATMS',
- None: 'SPACECRAFT'}
+instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', 'atms': 'ATMS', None: 'SPACECRAFT'}
diff --git a/edosl0util/timecode.py b/edosl0util/timecode.py
index 37f2a7507421ec518b321c9498b6dab1094696f2..8c2c54f1fc63fd5bf5b1216e2d36107fc14379b5 100644
--- a/edosl0util/timecode.py
+++ b/edosl0util/timecode.py
@@ -29,9 +29,9 @@ def timecode_parts_to_iet(days, ms, us, epoch):
"""
Convert components to a IET based on arbitrary epoch.
"""
- return _grain.utc2tai(
+ return int(_grain.utc2tai(
epoch + timedelta(days=days, milliseconds=ms, microseconds=us),
- epoch)
+ epoch) * 1e6)
def cds_to_iet(days, ms, us):
diff --git a/tests/test_rdrgen.py b/tests/test_rdrgen.py
index 8ee1c3a32fb1162631de83c333b8c85816c948b0..2fee2b70f654c0fd5a42b2ad20edd329a89dccad 100644
--- a/tests/test_rdrgen.py
+++ b/tests/test_rdrgen.py
@@ -68,7 +68,8 @@ def verify_rdr_reproduction(orig_file_name, tmp_dir):
p = subprocess.Popen(
['h5diff', '-c', '-p', '1e-5', orig_file, new_file],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- assert p.communicate()[0] == ''
+ stdout, _ = p.communicate()
+ assert not stdout
assert p.returncode == 0
@@ -172,7 +173,7 @@ class TestViirsGroupedPacketTimeTracker(object):
def run(pkt):
return m.iet_to_datetime(m.ViirsGroupedPacketTimeTracker.get_viirs_iet(pkt))
- with open(self.l0_path) as l0_file:
+ with open(self.l0_path, 'rb') as l0_file:
stream = jpss_packet_stream(l0_file)
standalone_pkt = next(p for p in stream if p.is_standalone())
first_pkt = next(p for p in stream if p.is_first())