diff --git a/edosl0util/crgen.py b/edosl0util/crgen.py
new file mode 100644
index 0000000000000000000000000000000000000000..087af19c24056e06cf6fe7166a725cd78898e4ea
--- /dev/null
+++ b/edosl0util/crgen.py
@@ -0,0 +1,200 @@
+
+"""EDOS PDS construction record generation for SUOMI NPP"""
+
+from datetime import datetime
+import itertools
+import os
+from edosl0util.headers import DaySegmentedTimecode
+from edosl0util.stream import jpss_packet_stream
+
+def test_build_apid_info():
+ # FIXME: build CR comparison into the CLI
+ calculated = build_cr('P1571289CRISSCIENCEAAT15320210920101.PDS')
+ ingested = read_pds_cr('P1571289CRISSCIENCEAAT15320210920100.PDS')
+ insert_fake_cr_info(ingested)
+ del calculated['completion_time'] # it seems CR completion time does not match PDS
+ del ingested['completion_time'] # creation time from the file name
+ assert calculated == ingested
+
+def build_cr(pds_file, prev_pds_file=None):
+ """Best-effort CR generation by scanning a PDS data file
+
+ Previous PDS data file may also be given to make gap detection more complete.
+ """
+ def main():
+ scan = scan_packets(pds_file, prev_pds_file)
+ rv = {}
+ rv['pds_id'] = pds_id_from_path(pds_file)
+ rv['completion_time'] = get_pds_creation_time(pds_file)
+ rv['first_packet_time'] = scan['first_packet_time']
+ rv['last_packet_time'] = scan['last_packet_time']
+ rv['apid_info'] = build_apid_info(scan['apid_info'])
+ rv['apid_count'] = len(rv['apid_info'])
+ rv['file_info'] = file_info(rv['pds_id'], rv['apid_info'])
+ rv['file_count'] = len(rv['file_info'])
+ rv.update(aggregated_values(rv['apid_info']))
+ insert_fake_cr_info(rv)
+ return rv
+ def file_info(pds_id, apid_info):
+ cr_entry = {'file_name': pds_id + '.PDS', 'apid_count': 0, 'apid_info' : []}
+ data_entry = {'file_name': pds_file,
+ 'apid_count': len(apid_info),
+ 'apid_info': [{'scid': npp_scid, 'apid': entry['apid'],
+ 'first_packet_time': entry['first_packet_time'],
+ 'last_packet_time': entry['last_packet_time']}
+ for entry in apid_info]}
+ return [cr_entry, data_entry]
+ def aggregated_values(apid_info):
+ keys = ['total_packets', 'total_bytes', 'gap_count', 'fill_bytes',
+ 'mismatched_length_packets', 'rs_corrected_packets']
+ return {key: sum(entry[key] for entry in apid_info) for key in keys}
+ return main()
+
+def insert_fake_cr_info(cr):
+ """Populate a CR with phony values for fields that can't be discovered via a packet scan"""
+ cr.update({'edos_sw_ver_major': 0, 'edos_sw_ver_minor': 0, 'cr_type': 1, 'test_flag': 0,
+ 'scs_count': 1, 'scs_info': {'start': missing_time_value,
+ 'stop': missing_time_value},
+ 'first_packet_esh_time': missing_time_value,
+ 'last_packet_esh_time': missing_time_value,
+ 'fill_bytes': 0, 'mismatched_length_packets': 0, 'rs_corrected_packets': 0})
+ insert_fake_apid_info(cr['apid_info'])
+
+def pds_id_from_path(pds_file):
+ """Pull 36-char PDS ID from a file name; that's the CR file name minus the .PDS"""
+ file_name = os.path.basename(pds_file)
+ pds_file_name_length = 40
+ if len(file_name) != pds_file_name_length:
+ raise ValueError('PDS file name {} not of expected length {}'.format(
+ file_name, pds_file_name_length))
+ return file_name[:34] + '00'
+
+def get_pds_creation_time(pds_file_or_id):
+ """Parse 11-char creation time out of a PDS ID or file name; return a DaySegmentedTimecode"""
+ return datetime_to_ccsds(datetime.strptime(pds_file_or_id[22:33], '%y%j%H%M%S'))
+
+def build_apid_info(scan_apid_info):
+ """Build up apid_info resulting from scan_packets into a full apid_info for a CR"""
+ for entry in scan_apid_info:
+ entry['scid'] = npp_scid
+ entry['vcid_count'] = 1
+ entry['vcid_info'] = [{'scid': npp_scid, 'vcid': npp_apid_to_vcid_map[entry['apid']]}]
+ entry['gap_count'] = len(entry['gap_info'])
+ del entry['last_packet_ssc'] # field was needed for bookkeeping but isn't in the CR
+ insert_fake_apid_info(apid_info)
+ return apid_info
+
+def insert_fake_apid_info(apid_info):
+ """Fill CR apid_info with phony values for fields that can't be found via a packet scan"""
+ for entry in apid_info:
+ entry.update({
+ 'fill_packets': 0, 'fill_packet_info': [], 'fill_bytes': 0,
+ 'mismatched_length_packets': 0, 'mismatched_length_packet_ssc_list': [],
+ 'first_packet_esh_time': missing_time_value,
+ 'last_packet_esh_time': missing_time_value,
+ 'rs_corrected_packets': 0})
+ for gap in entry['gap_info']:
+ gap.update({'pre_gap_packet_esh_time': missing_time_value,
+ 'post_gap_packet_esh_time': missing_time_value})
+
+missing_time_value = DaySegmentedTimecode(0, 0, 0)
+
+npp_scid = 157 # FIXME: i guess this should come from the file name
+
+# ripped from MDFCB, 2014-06-05 revision
+# modified to place CrIS FOV 6 in VCID 7 after testing against some real data
+npp_vcid_to_apids_map = {
+ 0: list(range(0, 15)) + list(range(16, 50)) + [65, 70, 100, 146, 155, 512, 513, 518,
+ 543, 544, 545, 550, 768, 769, 773] + list(range(1280, 1289)),
+ 1: [50, 101, 515, 528, 530, 531],
+ 2: [102],
+ 3: [103, 514, 536],
+ 4: [104],
+ 5: [105],
+ 6: [106, 1289, 1290] + list(set(range(1315, 1396)) - set(range(1318, 1391, 9))
+ - set(range(1320, 1393, 9))),
+ 7: [107] + list(range(1318, 1391, 9)) + list(range(1320, 1393, 9)),
+ 8: [108, 1294, 1295, 1296, 1398],
+ 9: [109],
+ 10: [110],
+ 11: [111, 560, 561, 564, 565, 576],
+ 12: [112, 562, 563, 566],
+ 13: [113, 546, 577, 578, 579, 580, 581, 582],
+ 14: [114],
+ 15: [115],
+ 16: [116] + list(range(800, 806)) + list(range(807, 824)) + [825, 826],
+ 17: [117, 806],
+ 18: [118, 770] + list(range(830, 854)) + [855, 856],
+ 19: [119],
+ 20: [120],
+ 21: [121, 517, 524, 549, 556, 780, 1291, 1292, 1293, 1397],
+ 22: [122],
+ 24: [147, 148, 149, 150]}
+npp_apid_to_vcid_map = {apid: vcid for vcid, apids in npp_vcid_to_apids_map.items()
+ for apid in apids}
+
+def scan_packets(pds_file, prev_pds_file=None):
+ """Scan a PDS data file for information needed to produce a construction record"""
+ def main():
+ prev_apid_map = build_prev_apid_map(prev_pds_file)
+ apid_map = {}
+ stream = jpss_packet_stream(open(pds_file, 'rb'))
+ first_pkt = stream.next()
+ for pkt in itertools.chain([first_pkt], stream):
+ entry = apid_map.get(pkt.apid)
+ if not entry:
+ entry_from_prev_pds = prev_apid_map.get(pkt.apid)
+ apid_map[pkt.apid] = init_entry(pkt, entry_from_prev_pds)
+ else:
+ update_entry(entry, pkt)
+ last_pkt = pkt
+ return {'first_packet_time': datetime_to_ccsds(first_pkt.stamp),
+ 'last_packet_time': datetime_to_ccsds(last_pkt.stamp),
+ 'apid_info': [apid_map[k] for k in sorted(apid_map)]}
+ def build_prev_apid_map(prev_pds_file):
+ if prev_pds_file:
+ return {entry['apid']: entry for entry in scan_packets(prev_pds_file)['apid_info']}
+ else:
+ return {}
+ def init_entry(pkt, entry_from_prev_pds):
+ rv = {'apid': pkt.apid,
+ 'first_packet_time': datetime_to_ccsds(pkt.stamp), 'first_packet_offset': pkt.offset,
+ 'last_packet_time': datetime_to_ccsds(pkt.stamp), 'last_packet_ssc': pkt.seqid,
+ 'total_packets': 1, 'total_bytes': pkt.size, 'gap_info': []}
+ if entry_from_prev_pds:
+ update_gap_info(rv['gap_info'], entry_from_prev_pds['last_packet_ssc'],
+ entry_from_prev_pds['last_packet_time'], pkt)
+ return rv
+ def update_entry(entry, new_pkt):
+ prev_last_ssc = entry['last_packet_ssc']
+ prev_last_time = entry['last_packet_time']
+ entry['last_packet_time'] = datetime_to_ccsds(new_pkt.stamp)
+ entry['last_packet_ssc'] = new_pkt.seqid
+ entry['total_packets'] += 1
+ entry['total_bytes'] += new_pkt.size
+ update_gap_info(entry['gap_info'], prev_last_ssc, prev_last_time, new_pkt)
+ def update_gap_info(gap_info, last_ssc, last_pkt_time, new_pkt):
+ ssc_limit = 16384 # one more than highest SSC
+ expected_new_ssc = (last_ssc + 1) % ssc_limit
+ if new_pkt.seqid != expected_new_ssc:
+ gap_entry = {'first_missing_ssc': expected_new_ssc,
+ 'missing_packet_count': (new_pkt.seqid - expected_new_ssc) % ssc_limit,
+ 'pre_gap_packet_time': last_pkt_time,
+ 'post_gap_packet_time': datetime_to_ccsds(new_pkt.stamp),
+ 'post_gap_packet_offset': new_pkt.offset}
+ gap_info.append(gap_entry)
+ return main()
+
+def datetime_to_ccsds(dt):
+ """Convert a packet stamp to DaySegmentedTimecode
+
+ Handles input of None by returning epoch value of 1958-01-01.
+ """
+ if dt is not None:
+ epoch = datetime(1958, 1, 1)
+ days = (dt - epoch).days
+ micros = int((dt - datetime(dt.year, dt.month, dt.day)).total_seconds() * 1e6)
+ return DaySegmentedTimecode(days, micros // 1000, micros % 1000)
+ else:
+ return DaySegmentedTimecode()
+
diff --git a/edosl0util/crio.py b/edosl0util/crio.py
new file mode 100644
index 0000000000000000000000000000000000000000..11e9439f5281bd329f067d7ce992801ea1fccda6
--- /dev/null
+++ b/edosl0util/crio.py
@@ -0,0 +1,186 @@
+
+"""PDS construction record input and output"""
+
+import ctypes as c
+import warnings
+from edosl0util.headers import BaseStruct, DaySegmentedTimecode
+
+def read(cr_file):
+ """Parse a PDS construction record from a file (*00.PDS)"""
+ def main():
+ rv = {}
+ with open(cr_file, 'rb') as f:
+ read_into_dict(f, Main1Struct, rv)
+ rv['scs_info'] = [read_struct(f, ScsStruct) for i in range(rv['scs_count'])]
+ read_into_dict(f, Main2Struct, rv)
+ rv['apid_info'] = []
+ for i in range(rv['apid_count']):
+ d = {}
+ read_into_dict(f, Apid1Struct, d)
+ d['vcid_info'] = [read_struct(f, ApidVcidStruct) for j in range(d['vcid_count'])]
+ read_into_dict(f, Apid2Struct, d)
+ d['gap_info'] = [read_struct(f, ApidGapStruct) for j in range(d['gap_count'])]
+ read_into_dict(f, Apid3Struct, d)
+ d['fill_packet_info'] = [
+ read_struct(f, ApidFillStruct) for j in range(d['fill_packets'])]
+ read_into_dict(f, Apid4Struct, d)
+ d['mismatched_length_packet_ssc_list'] = [
+ read_struct(f, ApidMismatchedLengthStruct)['packet_ssc']
+ for i in range(d['mismatched_length_packets'])]
+ read_into_dict(f, Apid5Struct, d)
+ rv['apid_info'].append(d)
+ read_into_dict(f, Main3Struct, rv)
+ rv['file_info'] = []
+ for i in range(rv['file_count']):
+ d = {}
+ read_into_dict(f, FileStruct, d)
+ d['apid_info'] = [read_struct(f, FileApidStruct) for j in range(d['apid_count'])]
+ if d['apid_count'] == 0: # bogus all-zero apid struct is present for the CR file
+ read_struct(f, FileApidStruct)
+ rv['file_info'].append(d)
+ if f.read():
+ warnings.warn('{} bytes remain after reading CR'.format(len(extra)))
+ return rv
+ def read_into_dict(f, struct, data):
+ data.update(read_struct(f, struct))
+ def read_struct(f, struct):
+ rv = struct_to_dict(struct.from_buffer_copy(f.read(c.sizeof(struct))))
+ rv = {k: v for k, v in rv.items() if not k.startswith('spare_')} # no spare fields
+ return {k: int(v) if isinstance(v, long) else v for k, v in rv.items()} # no longs
+ return main()
+
+def write(cr, out_file):
+ """Write out a PDS construction record file"""
+ def main():
+ with open(out_file, 'wb') as f:
+ write_struct(cr, Main1Struct, f)
+ for d in cr['scs_info']:
+ write_struct(d, ScsStruct, f)
+ write_struct(cr, Main2Struct, f)
+ for d in cr['apid_info']:
+ write_struct(d, Apid1Struct, f)
+ for dd in d['vcid_info']:
+ write_struct(dd, ApidVcidStruct, f)
+ write_struct(d, Apid2Struct, f)
+ for dd in d['gap_info']:
+ write_struct(dd, ApidGapStruct, f)
+ write_struct(d, Apid3Struct, f)
+ for dd in d['fill_packet_info']:
+ write_struct(dd, ApidFillStruct, f)
+ write_struct(d, Apid4Struct, f)
+ for ssc in d['mismatched_length_packet_ssc_list']:
+ write_struct({'ssc': ssc}, ApidMismatchedLengthStruct, f)
+ write_struct(d, Apid5Struct, f)
+ write_struct(cr, Main3Struct, f)
+ for d in cr['file_info']:
+ write_struct(d, FileStruct, f)
+ for dd in d['apid_info']:
+ write_struct(dd, FileApidStruct, f)
+ if d['apid_count'] == 0:
+ write_struct({}, FileApidStruct, f) # one all-zero apid struct if no others
+ def write_struct(data, struct, out):
+ fields = [f[0] for f in struct._fields_]
+ struct_data = {k: v for k, v in data.items() if k in fields}
+ out.write(memoryview(struct(**struct_data)))
+ main()
+
+def struct_to_dict(s):
+ return {f[0]: getattr(s, f[0]) for f in s._fields_}
+
+class Main1Struct(BaseStruct):
+ _fields_ = [('edos_sw_ver_major', c.c_uint8),
+ ('edos_sw_ver_minor', c.c_uint8),
+ ('cr_type', c.c_uint8),
+ ('spare_1', c.c_uint8 * 1),
+ ('pds_id', c.c_char * 36),
+ ('test_flag', c.c_uint8),
+ ('spare_2', c.c_uint8 * 9),
+ ('scs_count', c.c_uint16)]
+
+class ScsStruct(BaseStruct):
+ _fields_ = [('start', DaySegmentedTimecode),
+ ('stop', DaySegmentedTimecode)]
+
+class Main2Struct(BaseStruct):
+ _fields_ = [('fill_bytes', c.c_uint64),
+ ('mismatched_length_packets', c.c_uint32),
+ ('first_packet_time', DaySegmentedTimecode),
+ ('last_packet_time', DaySegmentedTimecode),
+ ('first_packet_esh_time', DaySegmentedTimecode),
+ ('last_packet_esh_time', DaySegmentedTimecode),
+ ('rs_corrected_packets', c.c_uint32),
+ ('total_packets', c.c_uint32),
+ ('total_bytes', c.c_uint64),
+ ('gap_count', c.c_uint32),
+ ('completion_time', DaySegmentedTimecode),
+ ('spare_3', c.c_uint8 * 7),
+ ('apid_count', c.c_uint8)]
+
+class Apid1Struct(BaseStruct):
+ _fields_ = [('spare_1', c.c_uint8 * 1),
+ ('scid', c.c_uint8),
+ ('apid', c.c_uint16),
+ ('first_packet_offset', c.c_uint64),
+ ('spare_2', c.c_uint8 * 3),
+ ('vcid_count', c.c_uint8)]
+
+class ApidVcidStruct(BaseStruct):
+ _fields_ = [('spare_1', c.c_uint16),
+ ('spare_2', c.c_uint16, 2),
+ ('scid', c.c_uint16, 8),
+ ('vcid', c.c_uint16, 6)]
+
+class Apid2Struct(BaseStruct):
+ _fields_ = [('gap_count', c.c_uint32)]
+
+class ApidGapStruct(BaseStruct):
+ _fields_ = [('first_missing_ssc', c.c_uint32),
+ ('post_gap_packet_offset', c.c_uint64),
+ ('missing_packet_count', c.c_uint32),
+ ('pre_gap_packet_time', DaySegmentedTimecode),
+ ('post_gap_packet_time', DaySegmentedTimecode),
+ ('pre_gap_packet_esh_time', DaySegmentedTimecode),
+ ('post_gap_packet_esh_time', DaySegmentedTimecode)]
+
+class Apid3Struct(BaseStruct):
+ _fields_ = [('fill_packets', c.c_uint32)]
+
+class ApidFillStruct(BaseStruct):
+ _fields_ = [('packet_ssc', c.c_uint32),
+ ('packet_offset', c.c_uint64),
+ ('first_fill_byte', c.c_uint32)]
+
+class Apid4Struct(BaseStruct):
+ _fields_ = [('fill_bytes', c.c_uint64),
+ ('mismatched_length_packets', c.c_uint32)]
+
+class ApidMismatchedLengthStruct(BaseStruct):
+ _fields_ = [('packet_ssc', c.c_uint32)]
+
+class Apid5Struct(BaseStruct):
+ _fields_ = [('first_packet_time', DaySegmentedTimecode),
+ ('last_packet_time', DaySegmentedTimecode),
+ ('first_packet_esh_time', DaySegmentedTimecode),
+ ('last_packet_esh_time', DaySegmentedTimecode),
+ ('rs_corrected_packets', c.c_uint32),
+ ('total_packets', c.c_uint32),
+ ('total_bytes', c.c_uint64),
+ ('spare_3', c.c_uint64)]
+
+class Main3Struct(BaseStruct):
+ _fields_ = [('spare_4', c.c_uint8 * 3),
+ ('file_count', c.c_uint8)]
+
+class FileStruct(BaseStruct):
+ _fields_ = [('file_name', c.c_char * 40),
+ ('spare_1', c.c_uint8 * 3),
+ ('apid_count', c.c_uint8)]
+
+class FileApidStruct(BaseStruct):
+ _fields_ = [('spare_1', c.c_uint8 * 1),
+ ('scid', c.c_uint8),
+ ('apid', c.c_uint16),
+ ('first_packet_time', DaySegmentedTimecode),
+ ('last_packet_time', DaySegmentedTimecode),
+ ('spare_2', c.c_uint8 * 4)]
+
diff --git a/edosl0util/headers.py b/edosl0util/headers.py
index bb017f7fd12a7c66908805554852393e975e4551..2eb2ff8e55cffc461bbf6705ab097dc5dfcef112 100644
--- a/edosl0util/headers.py
+++ b/edosl0util/headers.py
@@ -37,6 +37,9 @@ class BaseStruct(c.BigEndianStructure):
fields = ', '.join('%s=%s' % (f[0], repr(getattr(self, f[0]))) for f in self._fields_)
return '<%s (%s)>' % (self.__class__.__name__, fields)
+ def __eq__(self, other):
+ return all(getattr(self, f[0]) == getattr(other, f[0]) for f in self._fields_)
+
class PrimaryHeader(BaseStruct):
"""