From 7f4418029ade2bd1519c5d4676cf11c9c2925855 Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Thu, 18 Jan 2018 10:37:25 -0600
Subject: [PATCH] More great stuff for jpssrdr
---
edosl0util/cli/rdr2l0.py | 101 +++++-----------------------------
edosl0util/jpssrdr.py | 116 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 126 insertions(+), 91 deletions(-)
diff --git a/edosl0util/cli/rdr2l0.py b/edosl0util/cli/rdr2l0.py
index 9cbd024..9097684 100644
--- a/edosl0util/cli/rdr2l0.py
+++ b/edosl0util/cli/rdr2l0.py
@@ -3,6 +3,9 @@ Create a NASA Level0 PDS file from a set of NOAA IDPS RDR files. The output
PDS file will have packets sorted by time and apid. Any packets marked in
the RDR PacketTracker as containing fill are removed.
"""
+from edosl0util.jpssrdr import cris_hsk, cris_dwell, atms_dwell, atms_sci, viirs_sci, spacecraft, \
+ cris_sci, atms_hsk
+
__copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
import os
@@ -10,103 +13,15 @@ import glob
import logging
from datetime import datetime, timedelta
-from edosl0util import stream, merge, jpssrdr
from edosl0util.cli import util
LOG = logging
-satellite_to_scid = {
- 'snpp': 157,
- 'j01': 159,
-}
-
-
-def pdsfilename(product, created):
- if len(product) < 20:
- product = product + 'A'*(20-len(product))
- return '{}XT{:%y%j%H%M%S}001.PDS'.format(product, created)
-
-
def remove_files(files):
[os.remove(f) for f in files]
-def _do_dump(filepat, satellite, product, rdrs, start, end):
- product = 'P{}{}'.format(satellite_to_scid[satellite], product)
- for filepath in rdrs:
- LOG.info('dumping %s', filepath)
- jpssrdr.write_rdr_datasets(filepath, skipfill=True)
-
- # alphanumeric sorting to bootstrap final sort
- inputs = sorted(glob.glob(filepat))
- streams = [stream.jpss_packet_stream(open(f, 'rb')) for f in inputs]
-
- pdsname = pdsfilename(product, start)
- LOG.info('merging to %s', pdsname)
- with open(pdsname, 'wb') as dest:
- merge.merge(streams, output=dest, trunc_to=[start, end])
- return pdsname
-
-
-def cris_hsk(satellite, rdrs, start, end):
- product = '1280CRISHSK'
- return _do_dump('*.telemetry.pkts', satellite, product, rdrs, start, end)
-
-
-def cris_dwell(satellite, rdrs, start, end):
- product = '1291CRISDWELL'
- return _do_dump('*.dwell.pkts', satellite, product, rdrs, start, end)
-
-
-def cris_sci(satellite, rdrs, start, end):
- product = '1289CRISSCIENCE'
- return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
-
-
-def atms_hsk(satellite, rdrs, start, end):
- product = '0518ATMSHSK'
- return _do_dump('*.telemetry.pkts', satellite, product, rdrs, start, end)
-
-
-def atms_dwell(satellite, rdrs, start, end):
- product = '0517ATMSDWELL'
- return _do_dump('*.dwell.pkts', satellite, product, rdrs, start, end)
-
-
-def atms_sci(satellite, rdrs, start, end):
- product = '0515ATMSSCIENCE'
- return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
-
-
-def viirs_sci(satellite, rdrs, start, end):
- product = '0826VIIRSSCIENCE'
- return _do_dump('*.science.pkts', satellite, product, rdrs, start, end)
-
-
-def spacecraft(satellite, rdrs, start, end):
- product = 'SPACECRAFT'
- for filepath in rdrs:
- LOG.info('dumping %s', filepath)
- jpssrdr.write_rdr_datasets(filepath, ancillary=True, skipfill=True)
-
- scid = satellite_to_scid[satellite]
- for apid in (0, 8, 11):
- # alphanumeric sorting to bootstrap final sort
- inputs = sorted(glob.glob('*.ancillary{}.pkts'.format(apid)))
- files = [open(f, 'rb') for f in inputs]
- streams = [stream.jpss_packet_stream(f) for f in files]
-
- product = 'P{}{:04d}'.format(scid, apid)
- pdsname = pdsfilename(product, start)
- LOG.info('merging to %s', pdsname)
- with open(pdsname, 'wb') as dest:
- merge.merge(streams, output=dest, trunc_to=[start, end])
-
- # lots of files so make sure they're closed
- [f.close() for f in files]
-
-
def main():
# XXX: This currently uses standard CCSDS packet merging that does not have
# any context regarding fill packets. In the future if it is desired to
@@ -118,8 +33,10 @@ def main():
parser.add_argument(
'-S', '--satellite', choices=['snpp', 'j01'], default='snpp',
help='Satellite used to set SCID')
+
def timestamp(v):
return datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
+
parser.add_argument(
'-s', '--start', type=timestamp, required=True,
help=('File start time. Data before this time will be dropped. This '
@@ -136,12 +53,14 @@ def main():
def cmd_cris_hsk(args):
cris_hsk(args.satellite, args.rcrit, args.start, args.end)
+
subp = subs.add_parser('CRISHSK')
subp.add_argument('rcrit', nargs='+')
subp.set_defaults(func=cmd_cris_hsk)
def cmd_cris_dwell(args):
return cris_dwell(args.satellite, args.rdrs, args.start, args.end)
+
subp = subs.add_parser('CRISDWELL')
subp.add_argument(
'rdrs', nargs='+',
@@ -151,18 +70,21 @@ def main():
def cmd_cris_sci(args):
return cris_sci(args.satellite, args.rcris, args.start, args.end)
+
subp = subs.add_parser('CRISSCIENCE')
subp.add_argument('rcris', nargs='+')
subp.set_defaults(func=cmd_cris_sci)
def cmd_atms_hsk(args):
return atms_hsk(args.satellite, args.ratmt, args.start, args.end)
+
subp = subs.add_parser('ATMSHSK')
subp.add_argument('ratmt', nargs='+')
subp.set_defaults(func=cmd_atms_hsk)
def cmd_atms_dwell(args):
return atms_dwell(args.satellite, args.rdrs, args.start, args.end)
+
subp = subs.add_parser('ATMSDWELL')
subp.add_argument(
'rdrs', nargs='+',
@@ -172,18 +94,21 @@ def main():
def cmd_atms_sci(args):
return atms_sci(args.satellite, args.ratms, args.start, args.end)
+
subp = subs.add_parser('ATMSSCIENCE')
subp.add_argument('ratms', nargs='+')
subp.set_defaults(func=cmd_atms_sci)
def cmd_viirs_sci(args):
return viirs_sci(args.satellite, args.rvirs, args.start, args.end)
+
subp = subs.add_parser('VIIRSSCIENCE')
subp.add_argument('rvirs', nargs='+')
subp.set_defaults(func=cmd_viirs_sci)
def cmd_spacecraft(args):
return spacecraft(args.satellite, args.rnsca, args.start, args.end)
+
subp = subs.add_parser('SPACECRAFT')
subp.add_argument('rnsca', nargs='+')
subp.set_defaults(func=cmd_spacecraft)
diff --git a/edosl0util/jpssrdr.py b/edosl0util/jpssrdr.py
index 9601314..a22a413 100644
--- a/edosl0util/jpssrdr.py
+++ b/edosl0util/jpssrdr.py
@@ -7,6 +7,10 @@ Code for reading/writing/manipulating JPSS Common RDR files as documented in:
http://jointmission.gsfc.nasa.gov/sciencedocs/2015-06/474-00001-02_JPSS-CDFCB-X-Vol-II_0123B.pdf
"""
+import glob
+
+from edosl0util.stream import jpss_packet_stream
+
__copyright__ = "Copyright (C) 2015 University of Wisconsin SSEC. All rights reserved."
import ctypes as c
@@ -17,10 +21,18 @@ from collections import namedtuple
import numpy as np
from h5py import File as H5File
-from edosl0util.headers import BaseStruct
+from .headers import BaseStruct
+from .merge import merge
LOG = logging.getLogger(__name__)
+satellite_to_scid = {
+ 'npp': 157,
+ 'snpp': 157,
+ 'j01': 159,
+ 'jpss1': 159,
+}
+
class StaticHeader(BaseStruct):
"""
@@ -156,7 +168,8 @@ def _generate_packet_datasets(group):
yield name, np.array(ds)
-def _find_data_group(fobj, name, sensors=['viirs', 'cris', 'atms']):
+def _find_data_group(fobj, name, sensors=None):
+ sensors = sensors or ['viirs', 'cris', 'atms']
for sensor in sensors:
group = fobj.get('/All_Data/{}-{}-RDR_All'.format(sensor.upper(), name.upper()))
if group:
@@ -196,7 +209,7 @@ def _rdrs_for_packet_dataset(group):
try:
rdr = decode_rdr_blob(buf)
except ValueError as e:
- LOG.warn('{} ({})'.format(e.message, name))
+ LOG.warning('{} ({})'.format(e, name))
continue
rdrs.append(rdr)
return rdrs
@@ -290,3 +303,100 @@ def write_rdr_datasets(filepath, ancillary=False, skipfill=False):
rdr.header.type_id.decode())
_write_packets(rdr.packets(), dest, skipfill)
return rdrs
+
+
+def pdsfilename(product, created):
+ if len(product) < 20:
+ product = product + 'A' * (20 - len(product))
+ return '{}XT{:%y%j%H%M%S}001.PDS'.format(product, created)
+
+
+def _do_rdr_to_l0(filepat, satellite, product, rdrs, start, end):
+ product = 'P{}{}'.format(satellite_to_scid[satellite], product)
+ for filepath in rdrs:
+ LOG.info('dumping %s', filepath)
+ write_rdr_datasets(filepath, skipfill=True)
+
+ # alphanumeric sorting to bootstrap final sort
+ inputs = sorted(glob.glob(filepat))
+ streams = [jpss_packet_stream(open(f, 'rb')) for f in inputs]
+
+ pdsname = pdsfilename(product, start)
+ LOG.info('merging to %s', pdsname)
+ with open(pdsname, 'wb') as dest:
+ merge(streams, output=dest, trunc_to=[start, end])
+ return pdsname
+
+
+def cris_hsk_to_l0(satellite, rdrs, start, end):
+ product = '1280CRISHSK'
+ return _do_rdr_to_l0('*.telemetry.pkts', satellite, product, rdrs, start, end)
+
+
+def cris_dwell_to_l0(satellite, rdrs, start, end):
+ product = '1291CRISDWELL'
+ return _do_rdr_to_l0('*.dwell.pkts', satellite, product, rdrs, start, end)
+
+
+def cris_sci_to_l0(satellite, rdrs, start, end):
+ product = '1289CRISSCIENCE'
+ return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
+
+
+def atms_hsk_to_l0(satellite, rdrs, start, end):
+ product = '0518ATMSHSK'
+ return _do_rdr_to_l0('*.telemetry.pkts', satellite, product, rdrs, start, end)
+
+
+def atms_dwell_to_l0(satellite, rdrs, start, end):
+ product = '0517ATMSDWELL'
+ return _do_rdr_to_l0('*.dwell.pkts', satellite, product, rdrs, start, end)
+
+
+def atms_sci_to_l0(satellite, rdrs, start, end):
+ product = '0515ATMSSCIENCE'
+ return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
+
+
+def viirs_sci_to_l0(satellite, rdrs, start, end):
+ product = '0826VIIRSSCIENCE'
+ return _do_rdr_to_l0('*.science.pkts', satellite, product, rdrs, start, end)
+
+
+def spacecraft_to_l0(satellite, rdrs, start, end):
+ for filepath in rdrs:
+ LOG.info('dumping %s', filepath)
+ write_rdr_datasets(filepath, ancillary=True, skipfill=True)
+
+ filenames = []
+ scid = satellite_to_scid[satellite]
+ for apid in (0, 8, 11):
+ # alphanumeric sorting to bootstrap final sort
+ inputs = sorted(glob.glob('*.ancillary{}.pkts'.format(apid)))
+ files = [open(f, 'rb') for f in inputs]
+ streams = [jpss_packet_stream(f) for f in files]
+
+ product = 'P{}{:04d}'.format(scid, apid)
+ pdsname = pdsfilename(product, start)
+ LOG.info('merging to %s', pdsname)
+ with open(pdsname, 'wb') as dest:
+ merge(streams, output=dest, trunc_to=[start, end])
+ filenames.append(pdsname)
+
+ # lots of files so make sure they're closed
+ [f.close() for f in files]
+ return filenames
+
+
+def rdr_to_l0(satellite, rdrs, start, end):
+ filepaths = []
+ for product, filepath in {f.split('_', 1)[0]:f for f in rdrs}.items():
+ if 'RNSCA' in product:
+ filepaths += spacecraft_to_l0(satellite, [filepath], start, end)
+ if 'RVIRS' in product:
+ filepaths.append(viirs_sci_to_l0(satellite, [filepath], start, end))
+ if 'RCRIS' in product:
+ filepaths.append(cris_sci_to_l0(satellite, [filepath], start, end))
+ if 'RATMS' in product:
+ filepaths.append(atms_sci_to_l0(satellite, [filepath], start, end))
+ return filepaths
--
GitLab