Skip to content
Snippets Groups Projects
Commit 7f441802 authored by Bruce Flynn's avatar Bruce Flynn
Browse files

More great stuff for jpssrdr

parent 75fa96b8
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
edosl0util/cli/rdr2l0.py
+ 13
− 88
View file @ 7f441802
......@@ -3,6 +3,9 @@ Create a NASA Level0 PDS file from a set of NOAA IDPS RDR files. The output
PDS file will have packets sorted by time and apid. Any packets marked in
the RDR PacketTracker as containing fill are removed.
"""
from edosl0util.jpssrdr import cris_hsk, cris_dwell, atms_dwell, atms_sci, viirs_sci, spacecraft, \
cris_sci, atms_hsk
__copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
import os
......@@ -10,103 +13,15 @@ import glob
import logging
from datetime import datetime, timedelta
from edosl0util import stream, merge, jpssrdr
from edosl0util.cli import util
LOG = logging
satellite_to_scid = {
'snpp': 157,
'j01': 159,
}
def pdsfilename(product, created):
if len(product) < 20:
product = product + 'A'*(20-len(product))
return '{}XT{:%y%j%H%M%S}001.PDS'.format(product, created)
def remove_files(files):
[os.remove(f) for f in files]
def _do_dump(filepat, satellite, product, rdrs, start, end):
product = 'P{}{}'.format(satellite_to_scid[satellite], product)
for filepath in rdrs:
LOG.info('dumping %s', filepath)
jpssrdr.write_rdr_datasets(filepath, skipfill=True)
# alphanumeric sorting to bootstrap final sort
inputs = sorted(glob.glob(filepat))
streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
pdsname = pdsfilename(product, start)
LOG.info('merging to %s', pdsname)
with open(pdsname, 'wb') as dest:
merge.merge(streams, output=dest, trunc_to=[start, end])
return pdsname
def cris_hsk(satellite, rdrs, start, end):
product = '1280CRISHSK'
return _do_dump('*.telemetry.pkts', satellite, product, rdrs, start, end)
def cris_dwell(satellite, rdrs, start, end):
product = '1291CRISDWELL'
return _do_dump('*.dwell.pkts', satellite, product, rdrs, start, end)
def cris_sci(satellite, rdrs, start, end):
product = '1289CRISSCIENCE'
return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
def atms_hsk(satellite, rdrs, start, end):
product = '0518ATMSHSK'
return _do_dump('*.telemetry.pkts', satellite, product, rdrs, start, end)
def atms_dwell(satellite, rdrs, start, end):
product = '0517ATMSDWELL'
return _do_dump('*.dwell.pkts', satellite, product, rdrs, start, end)
def atms_sci(satellite, rdrs, start, end):
product = '0515ATMSSCIENCE'
return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
def viirs_sci(satellite, rdrs, start, end):
product = '0826VIIRSSCIENCE'
return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
def spacecraft(satellite, rdrs, start, end):
product = 'SPACECRAFT'
for filepath in rdrs:
LOG.info('dumping %s', filepath)
jpssrdr.write_rdr_datasets(filepath, ancillary=True, skipfill=True)
scid = satellite_to_scid[satellite]
for apid in (0, 8, 11):
# alphanumeric sorting to bootstrap final sort
inputs = sorted(glob.glob('*.ancillary{}.pkts'.format(apid)))
files = [open(f, 'rb') for f in inputs]
streams = [stream.jpss_packet_stream(f) for f in files]
product = 'P{}{:04d}'.format(scid, apid)
pdsname = pdsfilename(product, start)
LOG.info('merging to %s', pdsname)
with open(pdsname, 'wb') as dest:
merge.merge(streams, output=dest, trunc_to=[start, end])
# lots of files so make sure they're closed
[f.close() for f in files]
def main():
# XXX: This currently uses standard CCSDS packet merging that does not have
# any context regarding fill packets. In the future if it is desired to
......@@ -118,8 +33,10 @@ def main():
parser.add_argument(
'-S', '--satellite', choices=['snpp', 'j01'], default='snpp',
help='Satellite used to set SCID')
def timestamp(v):
return datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
parser.add_argument(
'-s', '--start', type=timestamp, required=True,
help=('File start time. Data before this time will be dropped. This '
......@@ -136,12 +53,14 @@ def main():
def cmd_cris_hsk(args):
cris_hsk(args.satellite, args.rcrit, args.start, args.end)
subp = subs.add_parser('CRISHSK')
subp.add_argument('rcrit', nargs='+')
subp.set_defaults(func=cmd_cris_hsk)
def cmd_cris_dwell(args):
return cris_dwell(args.satellite, args.rdrs, args.start, args.end)
subp = subs.add_parser('CRISDWELL')
subp.add_argument(
'rdrs', nargs='+',
......@@ -151,18 +70,21 @@ def main():
def cmd_cris_sci(args):
return cris_sci(args.satellite, args.rcris, args.start, args.end)
subp = subs.add_parser('CRISSCIENCE')
subp.add_argument('rcris', nargs='+')
subp.set_defaults(func=cmd_cris_sci)
def cmd_atms_hsk(args):
return atms_hsk(args.satellite, args.ratmt, args.start, args.end)
subp = subs.add_parser('ATMSHSK')
subp.add_argument('ratmt', nargs='+')
subp.set_defaults(func=cmd_atms_hsk)
def cmd_atms_dwell(args):
return atms_dwell(args.satellite, args.rdrs, args.start, args.end)
subp = subs.add_parser('ATMSDWELL')
subp.add_argument(
'rdrs', nargs='+',
......@@ -172,18 +94,21 @@ def main():
def cmd_atms_sci(args):
return atms_sci(args.satellite, args.ratms, args.start, args.end)
subp = subs.add_parser('ATMSSCIENCE')
subp.add_argument('ratms', nargs='+')
subp.set_defaults(func=cmd_atms_sci)
def cmd_viirs_sci(args):
return viirs_sci(args.satellite, args.rvirs, args.start, args.end)
subp = subs.add_parser('VIIRSSCIENCE')
subp.add_argument('rvirs', nargs='+')
subp.set_defaults(func=cmd_viirs_sci)
def cmd_spacecraft(args):
return spacecraft(args.satellite, args.rnsca, args.start, args.end)
subp = subs.add_parser('SPACECRAFT')
subp.add_argument('rnsca', nargs='+')
subp.set_defaults(func=cmd_spacecraft)
......
edosl0util/jpssrdr.py
+ 113
− 3
View file @ 7f441802
......@@ -7,6 +7,10 @@ Code for reading/writing/manipulating JPSS Common RDR files as documented in:
http://jointmission.gsfc.nasa.gov/sciencedocs/2015-06/474-00001-02_JPSS-CDFCB-X-Vol-II_0123B.pdf
"""
import glob
from edosl0util.stream import jpss_packet_stream
__copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
import ctypes as c
......@@ -17,10 +21,18 @@ from collections import namedtuple
import numpy as np
from h5py import File as H5File
from edosl0util.headers import BaseStruct
from .headers import BaseStruct
from .merge import merge
LOG = logging.getLogger(__name__)
satellite_to_scid = {
'npp': 157,
'snpp': 157,
'j01': 159,
'jpss1': 159,
}
class StaticHeader(BaseStruct):
"""
......@@ -156,7 +168,8 @@ def _generate_packet_datasets(group):
yield name, np.array(ds)
def _find_data_group(fobj, name, sensors=['viirs', 'cris', 'atms']):
def _find_data_group(fobj, name, sensors=None):
sensors = sensors or ['viirs', 'cris', 'atms']
for sensor in sensors:
group = fobj.get('/All_Data/{}-{}-RDR_All'.format(sensor.upper(), name.upper()))
if group:
......@@ -196,7 +209,7 @@ def _rdrs_for_packet_dataset(group):
try:
rdr = decode_rdr_blob(buf)
except ValueError as e:
LOG.warn('{} ({})'.format(e.message, name))
LOG.warning('{} ({})'.format(e, name))
continue
rdrs.append(rdr)
return rdrs
......@@ -290,3 +303,100 @@ def write_rdr_datasets(filepath, ancillary=False, skipfill=False):
rdr.header.type_id.decode())
_write_packets(rdr.packets(), dest, skipfill)
return rdrs
def pdsfilename(product, created):
if len(product) < 20:
product = product + 'A' * (20 - len(product))
return '{}XT{:%y%j%H%M%S}001.PDS'.format(product, created)
def _do_rdr_to_l0(filepat, satellite, product, rdrs, start, end):
product = 'P{}{}'.format(satellite_to_scid[satellite], product)
for filepath in rdrs:
LOG.info('dumping %s', filepath)
write_rdr_datasets(filepath, skipfill=True)
# alphanumeric sorting to bootstrap final sort
inputs = sorted(glob.glob(filepat))
streams = [jpss_packet_stream(open(f, 'rb')) for f in inputs]
pdsname = pdsfilename(product, start)
LOG.info('merging to %s', pdsname)
with open(pdsname, 'wb') as dest:
merge(streams, output=dest, trunc_to=[start, end])
return pdsname
def cris_hsk_to_l0(satellite, rdrs, start, end):
product = '1280CRISHSK'
return _do_rdr_to_l0('*.telemetry.pkts', satellite, product, rdrs, start, end)
def cris_dwell_to_l0(satellite, rdrs, start, end):
product = '1291CRISDWELL'
return _do_rdr_to_l0('*.dwell.pkts', satellite, product, rdrs, start, end)
def cris_sci_to_l0(satellite, rdrs, start, end):
product = '1289CRISSCIENCE'
return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
def atms_hsk_to_l0(satellite, rdrs, start, end):
product = '0518ATMSHSK'
return _do_rdr_to_l0('*.telemetry.pkts', satellite, product, rdrs, start, end)
def atms_dwell_to_l0(satellite, rdrs, start, end):
product = '0517ATMSDWELL'
return _do_rdr_to_l0('*.dwell.pkts', satellite, product, rdrs, start, end)
def atms_sci_to_l0(satellite, rdrs, start, end):
product = '0515ATMSSCIENCE'
return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
def viirs_sci_to_l0(satellite, rdrs, start, end):
product = '0826VIIRSSCIENCE'
return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
def spacecraft_to_l0(satellite, rdrs, start, end):
for filepath in rdrs:
LOG.info('dumping %s', filepath)
write_rdr_datasets(filepath, ancillary=True, skipfill=True)
filenames = []
scid = satellite_to_scid[satellite]
for apid in (0, 8, 11):
# alphanumeric sorting to bootstrap final sort
inputs = sorted(glob.glob('*.ancillary{}.pkts'.format(apid)))
files = [open(f, 'rb') for f in inputs]
streams = [jpss_packet_stream(f) for f in files]
product = 'P{}{:04d}'.format(scid, apid)
pdsname = pdsfilename(product, start)
LOG.info('merging to %s', pdsname)
with open(pdsname, 'wb') as dest:
merge(streams, output=dest, trunc_to=[start, end])
filenames.append(pdsname)
# lots of files so make sure they're closed
[f.close() for f in files]
return filenames
def rdr_to_l0(satellite, rdrs, start, end):
filepaths = []
for product, filepath in {f.split('_', 1)[0]:f for f in rdrs}.items():
if 'RNSCA' in product:
filepaths += spacecraft_to_l0(satellite, [filepath], start, end)
if 'RVIRS' in product:
filepaths.append(viirs_sci_to_l0(satellite, [filepath], start, end))
if 'RCRIS' in product:
filepaths.append(cris_sci_to_l0(satellite, [filepath], start, end))
if 'RATMS' in product:
filepaths.append(atms_sci_to_l0(satellite, [filepath], start, end))
return filepaths
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment