From 6ed2ab75acc907c2428f642022c76a8f4c63e4da Mon Sep 17 00:00:00 2001
From: Greg Quinn <greg.quinn@ssec.wisc.edu>
Date: Tue, 8 Dec 2015 15:44:46 -0600
Subject: [PATCH] Get crio and crgen to pass pep8
Except for the draconian line length limit (used
--max-line-length=99).
---
edosl0util/crgen.py | 41 +++++++++++++++++++++++++++++++----------
edosl0util/crio.py | 30 +++++++++++++++++++++++++++---
2 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/edosl0util/crgen.py b/edosl0util/crgen.py
index 087af19..048e610 100644
--- a/edosl0util/crgen.py
+++ b/edosl0util/crgen.py
@@ -7,6 +7,7 @@ import os
from edosl0util.headers import DaySegmentedTimecode
from edosl0util.stream import jpss_packet_stream
+
def test_build_apid_info():
# FIXME: build CR comparison into the CLI
calculated = build_cr('P1571289CRISSCIENCEAAT15320210920101.PDS')
@@ -16,11 +17,13 @@ def test_build_apid_info():
del ingested['completion_time'] # creation time from the file name
assert calculated == ingested
+
def build_cr(pds_file, prev_pds_file=None):
"""Best-effort CR generation by scanning a PDS data file
Previous PDS data file may also be given to make gap detection more complete.
"""
+
def main():
scan = scan_packets(pds_file, prev_pds_file)
rv = {}
@@ -35,31 +38,36 @@ def build_cr(pds_file, prev_pds_file=None):
rv.update(aggregated_values(rv['apid_info']))
insert_fake_cr_info(rv)
return rv
+
def file_info(pds_id, apid_info):
- cr_entry = {'file_name': pds_id + '.PDS', 'apid_count': 0, 'apid_info' : []}
+ cr_entry = {'file_name': pds_id + '.PDS', 'apid_count': 0, 'apid_info': []}
data_entry = {'file_name': pds_file,
'apid_count': len(apid_info),
'apid_info': [{'scid': npp_scid, 'apid': entry['apid'],
'first_packet_time': entry['first_packet_time'],
'last_packet_time': entry['last_packet_time']}
- for entry in apid_info]}
+ for entry in apid_info]}
return [cr_entry, data_entry]
+
def aggregated_values(apid_info):
keys = ['total_packets', 'total_bytes', 'gap_count', 'fill_bytes',
'mismatched_length_packets', 'rs_corrected_packets']
return {key: sum(entry[key] for entry in apid_info) for key in keys}
+
return main()
+
def insert_fake_cr_info(cr):
"""Populate a CR with phony values for fields that can't be discovered via a packet scan"""
cr.update({'edos_sw_ver_major': 0, 'edos_sw_ver_minor': 0, 'cr_type': 1, 'test_flag': 0,
'scs_count': 1, 'scs_info': {'start': missing_time_value,
- 'stop': missing_time_value},
+ 'stop': missing_time_value},
'first_packet_esh_time': missing_time_value,
'last_packet_esh_time': missing_time_value,
'fill_bytes': 0, 'mismatched_length_packets': 0, 'rs_corrected_packets': 0})
insert_fake_apid_info(cr['apid_info'])
+
def pds_id_from_path(pds_file):
"""Pull 36-char PDS ID from a file name; that's the CR file name minus the .PDS"""
file_name = os.path.basename(pds_file)
@@ -69,10 +77,12 @@ def pds_id_from_path(pds_file):
file_name, pds_file_name_length))
return file_name[:34] + '00'
+
def get_pds_creation_time(pds_file_or_id):
"""Parse 11-char creation time out of a PDS ID or file name; return a DaySegmentedTimecode"""
return datetime_to_ccsds(datetime.strptime(pds_file_or_id[22:33], '%y%j%H%M%S'))
+
def build_apid_info(scan_apid_info):
"""Build up apid_info resulting from scan_packets into a full apid_info for a CR"""
for entry in scan_apid_info:
@@ -84,6 +94,7 @@ def build_apid_info(scan_apid_info):
insert_fake_apid_info(apid_info)
return apid_info
+
def insert_fake_apid_info(apid_info):
"""Fill CR apid_info with phony values for fields that can't be found via a packet scan"""
for entry in apid_info:
@@ -97,6 +108,7 @@ def insert_fake_apid_info(apid_info):
gap.update({'pre_gap_packet_esh_time': missing_time_value,
'post_gap_packet_esh_time': missing_time_value})
+
missing_time_value = DaySegmentedTimecode(0, 0, 0)
npp_scid = 157 # FIXME: i guess this should come from the file name
@@ -104,15 +116,16 @@ npp_scid = 157 # FIXME: i guess this should come from the file name
# ripped from MDFCB, 2014-06-05 revision
# modified to place CrIS FOV 6 in VCID 7 after testing against some real data
npp_vcid_to_apids_map = {
- 0: list(range(0, 15)) + list(range(16, 50)) + [65, 70, 100, 146, 155, 512, 513, 518,
- 543, 544, 545, 550, 768, 769, 773] + list(range(1280, 1289)),
+ 0: (list(range(0, 15)) + list(range(16, 50)) +
+ [65, 70, 100, 146, 155, 512, 513, 518, 543, 544, 545, 550, 768, 769, 773] +
+ list(range(1280, 1289))),
1: [50, 101, 515, 528, 530, 531],
2: [102],
3: [103, 514, 536],
4: [104],
5: [105],
- 6: [106, 1289, 1290] + list(set(range(1315, 1396)) - set(range(1318, 1391, 9))
- - set(range(1320, 1393, 9))),
+ 6: [106, 1289, 1290] + list(set(range(1315, 1396)) - set(range(1318, 1391, 9)) -
+ set(range(1320, 1393, 9))),
7: [107] + list(range(1318, 1391, 9)) + list(range(1320, 1393, 9)),
8: [108, 1294, 1295, 1296, 1398],
9: [109],
@@ -130,11 +143,14 @@ npp_vcid_to_apids_map = {
21: [121, 517, 524, 549, 556, 780, 1291, 1292, 1293, 1397],
22: [122],
24: [147, 148, 149, 150]}
-npp_apid_to_vcid_map = {apid: vcid for vcid, apids in npp_vcid_to_apids_map.items()
- for apid in apids}
+npp_apid_to_vcid_map = {apid: vcid
+ for vcid, apids in npp_vcid_to_apids_map.items()
+ for apid in apids}
+
def scan_packets(pds_file, prev_pds_file=None):
"""Scan a PDS data file for information needed to produce a construction record"""
+
def main():
prev_apid_map = build_prev_apid_map(prev_pds_file)
apid_map = {}
@@ -151,11 +167,13 @@ def scan_packets(pds_file, prev_pds_file=None):
return {'first_packet_time': datetime_to_ccsds(first_pkt.stamp),
'last_packet_time': datetime_to_ccsds(last_pkt.stamp),
'apid_info': [apid_map[k] for k in sorted(apid_map)]}
+
def build_prev_apid_map(prev_pds_file):
if prev_pds_file:
return {entry['apid']: entry for entry in scan_packets(prev_pds_file)['apid_info']}
else:
return {}
+
def init_entry(pkt, entry_from_prev_pds):
rv = {'apid': pkt.apid,
'first_packet_time': datetime_to_ccsds(pkt.stamp), 'first_packet_offset': pkt.offset,
@@ -165,6 +183,7 @@ def scan_packets(pds_file, prev_pds_file=None):
update_gap_info(rv['gap_info'], entry_from_prev_pds['last_packet_ssc'],
entry_from_prev_pds['last_packet_time'], pkt)
return rv
+
def update_entry(entry, new_pkt):
prev_last_ssc = entry['last_packet_ssc']
prev_last_time = entry['last_packet_time']
@@ -173,6 +192,7 @@ def scan_packets(pds_file, prev_pds_file=None):
entry['total_packets'] += 1
entry['total_bytes'] += new_pkt.size
update_gap_info(entry['gap_info'], prev_last_ssc, prev_last_time, new_pkt)
+
def update_gap_info(gap_info, last_ssc, last_pkt_time, new_pkt):
ssc_limit = 16384 # one more than highest SSC
expected_new_ssc = (last_ssc + 1) % ssc_limit
@@ -183,8 +203,10 @@ def scan_packets(pds_file, prev_pds_file=None):
'post_gap_packet_time': datetime_to_ccsds(new_pkt.stamp),
'post_gap_packet_offset': new_pkt.offset}
gap_info.append(gap_entry)
+
return main()
+
def datetime_to_ccsds(dt):
"""Convert a packet stamp to DaySegmentedTimecode
@@ -197,4 +219,3 @@ def datetime_to_ccsds(dt):
return DaySegmentedTimecode(days, micros // 1000, micros % 1000)
else:
return DaySegmentedTimecode()
-
diff --git a/edosl0util/crio.py b/edosl0util/crio.py
index 11e9439..ba81f8b 100644
--- a/edosl0util/crio.py
+++ b/edosl0util/crio.py
@@ -5,8 +5,10 @@ import ctypes as c
import warnings
from edosl0util.headers import BaseStruct, DaySegmentedTimecode
+
def read(cr_file):
"""Parse a PDS construction record from a file (*00.PDS)"""
+
def main():
rv = {}
with open(cr_file, 'rb') as f:
@@ -26,7 +28,7 @@ def read(cr_file):
read_into_dict(f, Apid4Struct, d)
d['mismatched_length_packet_ssc_list'] = [
read_struct(f, ApidMismatchedLengthStruct)['packet_ssc']
- for i in range(d['mismatched_length_packets'])]
+ for i in range(d['mismatched_length_packets'])]
read_into_dict(f, Apid5Struct, d)
rv['apid_info'].append(d)
read_into_dict(f, Main3Struct, rv)
@@ -41,16 +43,21 @@ def read(cr_file):
if f.read():
warnings.warn('{} bytes remain after reading CR'.format(len(extra)))
return rv
+
def read_into_dict(f, struct, data):
data.update(read_struct(f, struct))
+
def read_struct(f, struct):
rv = struct_to_dict(struct.from_buffer_copy(f.read(c.sizeof(struct))))
rv = {k: v for k, v in rv.items() if not k.startswith('spare_')} # no spare fields
return {k: int(v) if isinstance(v, long) else v for k, v in rv.items()} # no longs
+
return main()
+
def write(cr, out_file):
"""Write out a PDS construction record file"""
+
def main():
with open(out_file, 'wb') as f:
write_struct(cr, Main1Struct, f)
@@ -78,15 +85,19 @@ def write(cr, out_file):
write_struct(dd, FileApidStruct, f)
if d['apid_count'] == 0:
write_struct({}, FileApidStruct, f) # one all-zero apid struct if no others
+
def write_struct(data, struct, out):
fields = [f[0] for f in struct._fields_]
struct_data = {k: v for k, v in data.items() if k in fields}
out.write(memoryview(struct(**struct_data)))
+
main()
+
def struct_to_dict(s):
return {f[0]: getattr(s, f[0]) for f in s._fields_}
+
class Main1Struct(BaseStruct):
_fields_ = [('edos_sw_ver_major', c.c_uint8),
('edos_sw_ver_minor', c.c_uint8),
@@ -97,10 +108,12 @@ class Main1Struct(BaseStruct):
('spare_2', c.c_uint8 * 9),
('scs_count', c.c_uint16)]
+
class ScsStruct(BaseStruct):
_fields_ = [('start', DaySegmentedTimecode),
('stop', DaySegmentedTimecode)]
+
class Main2Struct(BaseStruct):
_fields_ = [('fill_bytes', c.c_uint64),
('mismatched_length_packets', c.c_uint32),
@@ -116,6 +129,7 @@ class Main2Struct(BaseStruct):
('spare_3', c.c_uint8 * 7),
('apid_count', c.c_uint8)]
+
class Apid1Struct(BaseStruct):
_fields_ = [('spare_1', c.c_uint8 * 1),
('scid', c.c_uint8),
@@ -124,15 +138,18 @@ class Apid1Struct(BaseStruct):
('spare_2', c.c_uint8 * 3),
('vcid_count', c.c_uint8)]
+
class ApidVcidStruct(BaseStruct):
_fields_ = [('spare_1', c.c_uint16),
('spare_2', c.c_uint16, 2),
('scid', c.c_uint16, 8),
('vcid', c.c_uint16, 6)]
+
class Apid2Struct(BaseStruct):
_fields_ = [('gap_count', c.c_uint32)]
+
class ApidGapStruct(BaseStruct):
_fields_ = [('first_missing_ssc', c.c_uint32),
('post_gap_packet_offset', c.c_uint64),
@@ -141,22 +158,27 @@ class ApidGapStruct(BaseStruct):
('post_gap_packet_time', DaySegmentedTimecode),
('pre_gap_packet_esh_time', DaySegmentedTimecode),
('post_gap_packet_esh_time', DaySegmentedTimecode)]
-
+
+
class Apid3Struct(BaseStruct):
_fields_ = [('fill_packets', c.c_uint32)]
+
class ApidFillStruct(BaseStruct):
_fields_ = [('packet_ssc', c.c_uint32),
('packet_offset', c.c_uint64),
('first_fill_byte', c.c_uint32)]
+
class Apid4Struct(BaseStruct):
_fields_ = [('fill_bytes', c.c_uint64),
('mismatched_length_packets', c.c_uint32)]
+
class ApidMismatchedLengthStruct(BaseStruct):
_fields_ = [('packet_ssc', c.c_uint32)]
+
class Apid5Struct(BaseStruct):
_fields_ = [('first_packet_time', DaySegmentedTimecode),
('last_packet_time', DaySegmentedTimecode),
@@ -167,15 +189,18 @@ class Apid5Struct(BaseStruct):
('total_bytes', c.c_uint64),
('spare_3', c.c_uint64)]
+
class Main3Struct(BaseStruct):
_fields_ = [('spare_4', c.c_uint8 * 3),
('file_count', c.c_uint8)]
+
class FileStruct(BaseStruct):
_fields_ = [('file_name', c.c_char * 40),
('spare_1', c.c_uint8 * 3),
('apid_count', c.c_uint8)]
+
class FileApidStruct(BaseStruct):
_fields_ = [('spare_1', c.c_uint8 * 1),
('scid', c.c_uint8),
@@ -183,4 +208,3 @@ class FileApidStruct(BaseStruct):
('first_packet_time', DaySegmentedTimecode),
('last_packet_time', DaySegmentedTimecode),
('spare_2', c.c_uint8 * 4)]
-
--
GitLab