fix issues with rdrgen for py3. Fix issue with timecode utc to iet

edosl0util/compat.py

+str_types = (str,)
+try:
+    str_types += (unicode,)
+except:
+    pass

edosl0util/rdrgen.py

@@ -10,6 +10,7 @@ import h5py
 import numpy as np
 
 import edosl0util
+from edosl0util import compat
 from edosl0util.jpssrdr import (
     StaticHeader, Apid as ApidListItem, PacketTracker, decode_rdr_blob)
 from edosl0util.stream import jpss_packet_stream
@@ -19,7 +20,7 @@ from edosl0util.timecode import cds_to_iet, iet_to_dt
 def packets_to_rdrs(sat, l0_files, **kwargs):
     def iter_pkts(l0_files):
         for l0_file in l0_files:
-            with open(l0_file) as l0_file_obj:
+            with open(l0_file, 'rb') as l0_file_obj:
                 for pkt in jpss_packet_stream(l0_file_obj):
                     yield pkt
 
@@ -157,7 +158,7 @@ class BinnedTemporaryFileManager(object):
         if file_obj:
             file_obj.close()
         file_path = self._file_paths.pop(bin_key, '/dev/null')
-        file_obj = open(file_path)
+        file_obj = open(file_path, 'rb')
         try:
             yield file_obj
         finally:
@@ -272,6 +273,7 @@ class RdrWriter(object):
 
     @staticmethod
     def _set_h5_attrs(h5_obj, attrs):
+        attrs = _encodeall(attrs)
         # for some reason all the H5 attributes are in 2-D arrays in IDPS files
         for name, value in attrs.items():
             if isinstance(value, list):
@@ -300,6 +302,22 @@ class RdrWriter(object):
     _h5_regionref_dtype = h5py.special_dtype(ref=h5py.RegionReference)
 
 
+def _encodeall(v):
+    """
+    Recurse v encoding stringy values. dict keys are not encoded.
+    """
+    if isinstance(v, dict):
+        dtmp = {}
+        for k, v in v.items():
+            dtmp[k] = _encodeall(v)
+        return dtmp
+    if isinstance(v, list):
+        return [_encodeall(x) for x in v]
+    if isinstance(v, compat.str_types):
+        return v.encode()
+    return v
+
+
 def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
     get_jpss_packet_time = GetJpssPacketTime()
     granule_iet_end = granule_iet + rdr_type.gran_len
@@ -310,8 +328,9 @@ def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
     all_pkts = []
     for apid in rdr_type.apids:
         apid_info[apid.num] = {
-            'name': apid.name,
-            'pkts_reserved': apid.max_expected, 'pkts_received': 0,
+            'name': apid.name.encode(),
+            'pkts_reserved': apid.max_expected,
+            'pkts_received': 0,
             'first_tracker_index': total_trackers,
             'pkt_info': [{} for _ in range(apid.max_expected)]}
         total_trackers += apid.max_expected
@@ -340,9 +359,9 @@ def build_rdr_blob(sat, pkt_stream, rdr_type, granule_iet):
     buf = np.zeros([buf_size], np.uint8)  # zeros needed to null-pad strings
 
     header = StaticHeader.from_buffer(buf)
-    header.satellite = platform_short_names[sat]
-    header.sensor = instrument_short_names[rdr_type.sensor]
-    header.type_id = rdr_type.type_id
+    header.satellite = platform_short_names[sat].encode()
+    header.sensor = instrument_short_names[rdr_type.sensor].encode()
+    header.type_id = rdr_type.type_id.encode()
     header.num_apids = len(apid_info)
     header.apid_list_offset = apid_list_offset
     header.pkt_tracker_offset = pkt_tracker_offset
@@ -625,7 +644,7 @@ def make_rdr_filename(rdr_types, sat, aggr_begin, aggr_end, orbit_num, creation_
         origin = origin[:-1] + ('c' if compressed else 'u')
 
     def format_time(t):
-        return t.strftime('%H%M%S') + str(t.microsecond / 100000)
+        return t.strftime('%H%M%S') + str(t.microsecond // 100000)
 
     return '{p}_{s}_d{d:%Y%m%d}_t{b}_e{e}_b{n:05d}_c{c:%Y%m%d%H%M%S%f}_{o}_{m}.h5'.format(
         p=prod_ids, s=sat, d=aggr_begin, b=format_time(aggr_begin),
@@ -681,5 +700,4 @@ def get_overlapping_granules(sat, gran_len, start_iet, stop_iet):
 
 satellite_base_times = {'snpp': 1698019234000000}
 platform_short_names = {'snpp': 'NPP'}
-instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', 'atms': 'ATMS',
-                          None: 'SPACECRAFT'}
+instrument_short_names = {'viirs': 'VIIRS', 'cris': 'CrIS', 'atms': 'ATMS', None: 'SPACECRAFT'}

edosl0util/timecode.py

@@ -29,9 +29,9 @@ def timecode_parts_to_iet(days, ms, us, epoch):
     """
     Convert components to a IET based on arbitrary epoch.
     """
-    return _grain.utc2tai(
+    return int(_grain.utc2tai(
         epoch + timedelta(days=days, milliseconds=ms, microseconds=us),
-        epoch)
+        epoch) * 1e6)
 
 
 def cds_to_iet(days, ms, us):

tests/test_rdrgen.py

@@ -68,7 +68,8 @@ def verify_rdr_reproduction(orig_file_name, tmp_dir):
     p = subprocess.Popen(
         ['h5diff', '-c', '-p', '1e-5', orig_file, new_file],
         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    assert p.communicate()[0] == ''
+    stdout, _ = p.communicate()
+    assert not stdout
     assert p.returncode == 0
 
 
@@ -172,7 +173,7 @@ class TestViirsGroupedPacketTimeTracker(object):
         def run(pkt):
             return m.iet_to_datetime(m.ViirsGroupedPacketTimeTracker.get_viirs_iet(pkt))
 
-        with open(self.l0_path) as l0_file:
+        with open(self.l0_path, 'rb') as l0_file:
             stream = jpss_packet_stream(l0_file)
             standalone_pkt = next(p for p in stream if p.is_standalone())
             first_pkt = next(p for p in stream if p.is_first())