Skip to content
Snippets Groups Projects
Commit 1de5b7a8 authored by Greg Quinn's avatar Greg Quinn
Browse files

rdrgen: produce correct file names for RDRs

parent 2ee9754b
No related branches found
No related tags found
No related merge requests found
import ctypes import ctypes
import itertools import itertools
import os
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime from datetime import datetime
...@@ -13,8 +14,8 @@ from edosl0util.jpssrdr import ( ...@@ -13,8 +14,8 @@ from edosl0util.jpssrdr import (
StaticHeader, Apid as ApidListItem, PacketTracker, decode_rdr_blob) StaticHeader, Apid as ApidListItem, PacketTracker, decode_rdr_blob)
def write_rdr(sat, blob, def write_rdr(sat, blob, dir_path='.',
distributor=None, origin=None, domain=None, orbit_number=0, distributor=None, origin=None, domain=None, compressed=False, orbit_num=0,
creation_time=None, gran_creation_time=None, software_ver=None): creation_time=None, gran_creation_time=None, software_ver=None):
# TODO: write out the user block too?? # TODO: write out the user block too??
distributor = distributor or origin or default_origin distributor = distributor or origin or default_origin
...@@ -34,7 +35,9 @@ def write_rdr(sat, blob, ...@@ -34,7 +35,9 @@ def write_rdr(sat, blob,
gran_ver = 'A1' gran_ver = 'A1'
format_date = lambda t: t.strftime('%Y%m%d') format_date = lambda t: t.strftime('%Y%m%d')
format_time = lambda t: t.strftime('%H%M%S.%fZ') format_time = lambda t: t.strftime('%H%M%S.%fZ')
with h5py.File('rdr.h5', 'w') as h5_file: file_name = make_rdr_filename(rdr_type.product_id, sat, gran_time, gran_end_time,
orbit_num, creation_time, origin, domain, compressed)
with h5py.File(os.path.join(dir_path, file_name), 'w') as h5_file:
set_h5_attrs(h5_file, { set_h5_attrs(h5_file, {
'Distributor': distributor, 'Distributor': distributor,
'Mission_Name': 'S-NPP/JPSS', 'Mission_Name': 'S-NPP/JPSS',
...@@ -61,7 +64,7 @@ def write_rdr(sat, blob, ...@@ -61,7 +64,7 @@ def write_rdr(sat, blob,
'Beginning_Time': format_time(gran_time), 'Beginning_Time': format_time(gran_time),
'Ending_Date': format_date(gran_end_time), 'Ending_Date': format_date(gran_end_time),
'Ending_Time': format_time(gran_end_time), 'Ending_Time': format_time(gran_end_time),
'N_Beginning_Orbit_Number': np.uint64(orbit_number), # FIXME: detect? 'N_Beginning_Orbit_Number': np.uint64(orbit_num), # FIXME: detect?
'N_Beginning_Time_IET': np.uint64(gran_iet), 'N_Beginning_Time_IET': np.uint64(gran_iet),
'N_Creation_Date': format_date(gran_creation_time), 'N_Creation_Date': format_date(gran_creation_time),
'N_Creation_Time': format_time(gran_creation_time), 'N_Creation_Time': format_time(gran_creation_time),
...@@ -84,13 +87,14 @@ def write_rdr(sat, blob, ...@@ -84,13 +87,14 @@ def write_rdr(sat, blob,
set_h5_attrs(aggr_ds, { set_h5_attrs(aggr_ds, {
'AggregateBeginningDate': format_date(gran_time), 'AggregateBeginningDate': format_date(gran_time),
'AggregateBeginningGranuleID': gran_id, 'AggregateBeginningGranuleID': gran_id,
'AggregateBeginningOrbitNumber': np.uint64(orbit_number), 'AggregateBeginningOrbitNumber': np.uint64(orbit_num),
'AggregateBeginningTime': format_time(gran_time), 'AggregateBeginningTime': format_time(gran_time),
'AggregateEndingDate': format_date(gran_end_time), 'AggregateEndingDate': format_date(gran_end_time),
'AggregateEndingGranuleID': gran_id, 'AggregateEndingGranuleID': gran_id,
'AggregateEndingOrbitNumber': np.uint64(orbit_number), 'AggregateEndingOrbitNumber': np.uint64(orbit_num),
'AggregateEndingTime': format_time(gran_end_time), 'AggregateEndingTime': format_time(gran_end_time),
'AggregateNumberGranules': np.uint64(1)}) 'AggregateNumberGranules': np.uint64(1)})
return file_name
def set_h5_attrs(h5_obj, attrs): def set_h5_attrs(h5_obj, attrs):
...@@ -227,6 +231,7 @@ class ApidSpec(object): ...@@ -227,6 +231,7 @@ class ApidSpec(object):
class CrisScienceRdrType(object): class CrisScienceRdrType(object):
product_id = 'RCRIS'
short_name = 'CRIS-SCIENCE-RDR' short_name = 'CRIS-SCIENCE-RDR'
gran_len = 31997000 gran_len = 31997000
sensor = 'cris' sensor = 'cris'
...@@ -236,6 +241,7 @@ class CrisScienceRdrType(object): ...@@ -236,6 +241,7 @@ class CrisScienceRdrType(object):
class SpacecraftDiaryRdrType(object): class SpacecraftDiaryRdrType(object):
product_id = 'RNSCA'
short_name = 'SPACECRAFT-DIARY-RDR' short_name = 'SPACECRAFT-DIARY-RDR'
gran_len = 20000000 gran_len = 20000000
sensor = None sensor = None
...@@ -246,6 +252,22 @@ class SpacecraftDiaryRdrType(object): ...@@ -246,6 +252,22 @@ class SpacecraftDiaryRdrType(object):
ApidSpec(11, 'DIARY', max_expected=21)] ApidSpec(11, 'DIARY', max_expected=21)]
def make_rdr_filename(prod_id, sat, gran_time, gran_end_time, orbit_num, creation_time,
origin, domain, compressed):
sat_strs = {'snpp': 'npp'}
def format_gran_time(t):
return t.strftime('%H%M%S') + str(t.microsecond / 100000)
def format_origin(o):
if o.endswith('-'):
return o[:-1] + ('c' if compressed else 'u')
else:
return o
return '{p}_{s}_d{d:%Y%m%d}_t{b}_e{e}_b{n}_c{c:%Y%m%d%H%M%S%f}_{o}_{m}.h5'.format(
p=prod_id, s=sat_strs[sat], d=gran_time, b=format_gran_time(gran_time),
e=format_gran_time(gran_end_time), n=orbit_num, c=creation_time,
o=format_origin(origin), m=domain)
def timecode_to_iet(timecode): def timecode_to_iet(timecode):
# FIXME: move to timecode.py # FIXME: move to timecode.py
ccsds_epoch = Time('1958-01-01', scale='tai') ccsds_epoch = Time('1958-01-01', scale='tai')
......
...@@ -50,14 +50,21 @@ def test_can_reproduce_rdr_from_class(): ...@@ -50,14 +50,21 @@ def test_can_reproduce_rdr_from_class():
# generate new RDR from packets, injecting matching metadata from CLASS file # generate new RDR from packets, injecting matching metadata from CLASS file
blob = m.build_rdr_blob('snpp', jpss_packet_stream(StringIO(pkt_buf.tobytes()))) blob = m.build_rdr_blob('snpp', jpss_packet_stream(StringIO(pkt_buf.tobytes())))
m.write_rdr('snpp', blob, distributor='arch', origin='nob-', domain='ops', tmp_dir = '/tmp'
creation_time=datetime(2017, 9, 13, 22, 3, 40, 173580), file_name = m.write_rdr(
gran_creation_time=datetime(2017, 9, 12, 1, 37, 43, 474383), 'snpp', blob, tmp_dir,
orbit_number=30441, software_ver='I2.0.03.00') distributor='arch', origin='nob-', domain='ops',
creation_time=datetime(2017, 9, 13, 22, 3, 40, 173580),
gran_creation_time=datetime(2017, 9, 12, 1, 37, 43, 474383),
orbit_num=30441, software_ver='I2.0.03.00')
# file names should be identical
assert file_name == class_rdr_file
# use h5diff to verify files match. -p option is needed to allow some slop # use h5diff to verify files match. -p option is needed to allow some slop
# in comparing N_Percent_Missing_Data # in comparing N_Percent_Missing_Data
p = subprocess.Popen(['h5diff', '-c', '-p', '1e-6', class_rdr_path, 'rdr.h5'], p = subprocess.Popen(
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ['h5diff', '-c', '-p', '1e-6', class_rdr_path, os.path.join(tmp_dir, file_name)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
assert p.communicate()[0] == '' assert p.communicate()[0] == ''
assert p.returncode == 0 assert p.returncode == 0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment