Skip to content
Snippets Groups Projects
Commit b5cfb07f authored by Geoff Cureton's avatar Geoff Cureton
Browse files

Update fusion_matlab glue code to use sipsprod.

parent 4ebb8861
No related branches found
No related tags found
No related merge requests found
import sys
from datetime import datetime, timedelta
import traceback
import logging
from flo.time import TimeInterval
from timeutil import TimeInterval, datetime, timedelta
from flo.ui import local_prepare, local_execute
from flo.sw.fusion_matlab import FUSION_MATLAB
from flo.sw.fusion_matlab.utils import setup_logging
# every module should have a LOG object
LOG = logging.getLogger(__file__)
LOG = logging.getLogger(__name__)
comp = FUSION_MATLAB()
......@@ -19,15 +18,16 @@ comp = FUSION_MATLAB()
#
# General information
granule = datetime(2015, 4, 17, 14)
interval = TimeInterval(granule, granule+timedelta(minutes=0))
#granule = datetime(2015, 4, 17, 14)
#interval = TimeInterval(granule, granule+timedelta(minutes=0))
satellite = 'snpp'
#satellite = 'aqua'
delivery_id = '20170920-1'
version = '1.0dev1'
def local_execute_example(interval, satellite, delivery_id, skip_prepare=False, skip_execute=False, verbosity=2):
def local_execute_example(interval, satellite, version, skip_prepare=False, skip_execute=False, verbosity=2):
setup_logging(verbosity)
......@@ -39,7 +39,7 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
LOG.error("Invalid satellite.")
# Get the required context...
contexts = comp.find_contexts(interval, satellite, delivery_id)
contexts = comp.find_contexts(interval, satellite, version)
if len(contexts) != 0:
LOG.info("Candidate contexts in interval...")
......@@ -49,9 +49,11 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
try:
if not skip_prepare:
LOG.info("Running fusion_matlab local_prepare()...")
local_prepare(comp, contexts[0])
LOG.info("Preparing context... {}".format(contexts[0]))
local_prepare(comp, contexts[0])
if not skip_execute:
LOG.info("Running local_execute()...")
LOG.info("Running context... {}".format(contexts[0]))
local_execute(comp, contexts[0])
except Exception, err:
LOG.error("{}".format(err))
......@@ -59,7 +61,7 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
else:
LOG.error("There are no valid {} contexts for the interval {}.".format(satellite, interval))
def print_contexts(interval, satellite, delivery_id):
contexts = comp.find_contexts(interval, satellite, delivery_id)
def print_contexts(interval, satellite, version):
contexts = comp.find_contexts(interval, satellite, version)
for context in contexts:
print context
This diff is collapsed.
......@@ -5,6 +5,7 @@
import os
import sys
import numpy as np
from pyhdf.SD import SD, SDC
import netCDF4 as nc4
#from scipy.io import loadmat
import logging
......@@ -20,7 +21,7 @@ def detect(threshold, tru, tst):
"""Compute RMS error and return pass/fail boolean"""
yisq = (tst.ravel() - tru.ravel()) ** 2.0
rms = np.sqrt(np.sum(yisq) / float(len(yisq)))
LOG.info("RMS error is {} (threshold {})".format(rms, threshold))
LOG.debug("RMS error is {} (threshold {})".format(rms, threshold))
return bool(rms <= threshold), rms
......@@ -33,8 +34,8 @@ def detect(threshold, tru, tst):
#yield rad
def modis_band(filename, band_number=DEFAULT_BAND):
"""Yield M or F from a single MODIS or MODIS-AIRS fusion file"""
def modis_band_nc(filename, band_number=DEFAULT_BAND):
"""Yield M or F from a single MODIS or MODIS-AIRS fusion file, NetCDF4"""
LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
nc = nc4.Dataset(filename)
em = nc.variables['EV_1KM_Emissive']
......@@ -42,6 +43,26 @@ def modis_band(filename, band_number=DEFAULT_BAND):
band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
yield band(em, band_number)
def modis_band(filename, band_number=DEFAULT_BAND):
"""Yield M or F from a single MODIS or MODIS-AIRS fusion file, using HDF4"""
LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
print(filename)
file_obj = SD(str(filename))
em = file_obj.select('EV_1KM_Emissive')
bn,sc,of = dict(((b,i) for (i,b) in enumerate(map(int, em.band_names.split(','))))), em.radiance_scales, em.radiance_offsets
band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
yield band(em, band_number)
def modis_band_new(filename, band_number=DEFAULT_BAND):
"""Yield M or F from a single MODIS or MODIS-AIRS fusion file, using HDF4"""
LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
print(filename)
file_obj = SD(str(filename))
em = file_obj.select('EV_1KM_Emissive')
bn,sc,of = dict(((b,i) for (i,b) in enumerate(map(int, em.band_names.split(','))))), em.radiance_scales, em.radiance_offsets
band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
yield band(em, band_number)
def viirs_band(filename, band_number=DEFAULT_BAND):
"""Yield both Mband and Fband from a single file"""
......@@ -53,6 +74,14 @@ def viirs_band(filename, band_number=DEFAULT_BAND):
f = ncob['Fusion%02d' % band_number][:]
yield f
def viirs_band_new(filename, band_number=DEFAULT_BAND):
"""Yield both Mband and Fband from a single file"""
LOG.info("reading band {} (M and Fusion) from {} VIIRS NetCDF".format(band_number, filename))
nc = nc4.Dataset(filename)
ncob = nc['observation_data']
m = ncob['M%02d' % band_number][:]
f = ncob['Fusion%02d' % band_number][:]
yield {'M{}'.format(band_number):[m,f]}
def _debug(type_, value, tb):
"""enable with sys.excepthook = debug"""
......@@ -67,6 +96,7 @@ def _debug(type_, value, tb):
#SFX = {'.mat': mat_band, '.hdf': modis_band, '.nc': viirs_band}
SFX = {'.hdf': modis_band, '.nc': viirs_band}
SFX_new = {'.hdf': modis_band, '.nc': viirs_band_new}
def file_QC(input_rms, band, input_files):
generators = list(SFX[os.path.splitext(p)[-1].lower()](p, band) for p in input_files)
......
......@@ -11,7 +11,7 @@ from flo.sw.fusion_matlab import FUSION_MATLAB
from flo.sw.fusion_matlab.utils import setup_logging
# every module should have a LOG object
LOG = logging.getLogger(__file__)
LOG = logging.getLogger(__name__)
setup_logging(2)
......@@ -20,6 +20,7 @@ comp = FUSION_MATLAB()
satellite = 'aqua'
#satellite = 'snpp'
version = '1.0dev0'
delivery_id = '20170920-1'
# Specify the intervals
......
......@@ -24,7 +24,7 @@ import traceback
import logging
# every module should have a LOG object
LOG = logging.getLogger(__file__)
LOG = logging.getLogger(__name__)
def setup_logging(verbosity):
LOG.debug("Verbosity is {}".format(verbosity))
......
......@@ -24,7 +24,7 @@ import traceback
import logging
# every module should have a LOG object
LOG = logging.getLogger(__file__)
LOG = logging.getLogger(__name__)
def setup_logging(verbosity):
LOG.debug("Verbosity is {}".format(verbosity))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment