Update fusion_matlab glue code to use sipsprod.

example_local_prepare.py

 import sys
-from datetime import datetime, timedelta
 import traceback
 import logging
 
-from flo.time import TimeInterval
+from timeutil import TimeInterval, datetime, timedelta
 from flo.ui import local_prepare, local_execute
 
 from flo.sw.fusion_matlab import FUSION_MATLAB
 from flo.sw.fusion_matlab.utils import setup_logging
 
 # every module should have a LOG object
-LOG = logging.getLogger(__file__)
+LOG = logging.getLogger(__name__)
 
 comp = FUSION_MATLAB()
 
@@ -19,15 +18,16 @@ comp = FUSION_MATLAB()
 #
 
 # General information
-granule = datetime(2015, 4, 17, 14)
-interval = TimeInterval(granule, granule+timedelta(minutes=0))
+#granule = datetime(2015, 4, 17, 14)
+#interval = TimeInterval(granule, granule+timedelta(minutes=0))
 
 satellite = 'snpp'
 #satellite = 'aqua'
 delivery_id = '20170920-1'
+version = '1.0dev1'
 
 
-def local_execute_example(interval, satellite, delivery_id, skip_prepare=False, skip_execute=False, verbosity=2):
+def local_execute_example(interval, satellite, version, skip_prepare=False, skip_execute=False, verbosity=2):
 
     setup_logging(verbosity)
 
@@ -39,7 +39,7 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
         LOG.error("Invalid satellite.")
 
     # Get the required context...
-    contexts =  comp.find_contexts(interval, satellite, delivery_id)
+    contexts =  comp.find_contexts(interval, satellite, version)
 
     if len(contexts) != 0:
         LOG.info("Candidate contexts in interval...")
@@ -49,9 +49,11 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
         try:
             if not skip_prepare:
                 LOG.info("Running fusion_matlab local_prepare()...")
-                local_prepare(comp, contexts[0]) 
+                LOG.info("Preparing context... {}".format(contexts[0]))
+                local_prepare(comp, contexts[0])
             if not skip_execute:
                 LOG.info("Running local_execute()...")
+                LOG.info("Running context... {}".format(contexts[0]))
                 local_execute(comp, contexts[0])
         except Exception, err:
             LOG.error("{}".format(err))
@@ -59,7 +61,7 @@ def local_execute_example(interval, satellite, delivery_id, skip_prepare=False,
     else:
         LOG.error("There are no valid {} contexts for the interval {}.".format(satellite, interval))
 
-def print_contexts(interval, satellite, delivery_id):
-    contexts = comp.find_contexts(interval, satellite, delivery_id)
+def print_contexts(interval, satellite, version):
+    contexts = comp.find_contexts(interval, satellite, version)
     for context in contexts:
         print context

source/flo/detect_bad_fusion.py

@@ -5,6 +5,7 @@
 import os
 import sys
 import numpy as np
+from pyhdf.SD import SD, SDC
 import netCDF4 as nc4
 #from scipy.io import loadmat
 import logging
@@ -20,7 +21,7 @@ def detect(threshold, tru, tst):
     """Compute RMS error and return pass/fail boolean"""
     yisq = (tst.ravel() - tru.ravel()) ** 2.0
     rms = np.sqrt(np.sum(yisq) / float(len(yisq)))
-    LOG.info("RMS error is {} (threshold {})".format(rms, threshold))
+    LOG.debug("RMS error is {} (threshold {})".format(rms, threshold))
     return bool(rms <= threshold), rms
 
 
@@ -33,8 +34,8 @@ def detect(threshold, tru, tst):
     #yield rad
 
 
-def modis_band(filename, band_number=DEFAULT_BAND):
-    """Yield M or F from a single MODIS or MODIS-AIRS fusion file"""
+def modis_band_nc(filename, band_number=DEFAULT_BAND):
+    """Yield M or F from a single MODIS or MODIS-AIRS fusion file, NetCDF4"""
     LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
     nc = nc4.Dataset(filename)
     em = nc.variables['EV_1KM_Emissive']
@@ -42,6 +43,26 @@ def modis_band(filename, band_number=DEFAULT_BAND):
     band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
     yield band(em, band_number)
 
+def modis_band(filename, band_number=DEFAULT_BAND):
+    """Yield M or F from a single MODIS or MODIS-AIRS fusion file, using HDF4"""
+    LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
+    print(filename)
+    file_obj = SD(str(filename))
+    em = file_obj.select('EV_1KM_Emissive')
+    bn,sc,of = dict(((b,i) for (i,b) in enumerate(map(int, em.band_names.split(','))))), em.radiance_scales, em.radiance_offsets
+    band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
+    yield band(em, band_number)
+
+def modis_band_new(filename, band_number=DEFAULT_BAND):
+    """Yield M or F from a single MODIS or MODIS-AIRS fusion file, using HDF4"""
+    LOG.info("reading band {} from {} MODIS HDF".format(band_number, filename))
+    print(filename)
+    file_obj = SD(str(filename))
+    em = file_obj.select('EV_1KM_Emissive')
+    bn,sc,of = dict(((b,i) for (i,b) in enumerate(map(int, em.band_names.split(','))))), em.radiance_scales, em.radiance_offsets
+    band = lambda v, b: sc[bn[b]] * (v[bn[b]] - of[bn[b]])
+    yield band(em, band_number)
+
 
 def viirs_band(filename, band_number=DEFAULT_BAND):
     """Yield both Mband and Fband from a single file"""
@@ -53,6 +74,14 @@ def viirs_band(filename, band_number=DEFAULT_BAND):
     f = ncob['Fusion%02d' % band_number][:]
     yield f
 
+def viirs_band_new(filename, band_number=DEFAULT_BAND):
+    """Yield both Mband and Fband from a single file"""
+    LOG.info("reading band {} (M and Fusion) from {} VIIRS NetCDF".format(band_number, filename))
+    nc = nc4.Dataset(filename)
+    ncob = nc['observation_data']
+    m = ncob['M%02d' % band_number][:]
+    f = ncob['Fusion%02d' % band_number][:]
+    yield {'M{}'.format(band_number):[m,f]}
 
 def _debug(type_, value, tb):
     """enable with sys.excepthook = debug"""
@@ -67,6 +96,7 @@ def _debug(type_, value, tb):
 
 #SFX = {'.mat': mat_band, '.hdf': modis_band, '.nc': viirs_band}
 SFX = {'.hdf': modis_band, '.nc': viirs_band}
+SFX_new = {'.hdf': modis_band, '.nc': viirs_band_new}
 
 def file_QC(input_rms, band, input_files):
     generators = list(SFX[os.path.splitext(p)[-1].lower()](p, band) for p in input_files)

submit_fusion_matlab.py

@@ -11,7 +11,7 @@ from flo.sw.fusion_matlab import FUSION_MATLAB
 from flo.sw.fusion_matlab.utils import setup_logging
 
 # every module should have a LOG object
-LOG = logging.getLogger(__file__)
+LOG = logging.getLogger(__name__)
 
 setup_logging(2)
 
@@ -20,6 +20,7 @@ comp = FUSION_MATLAB()
 satellite = 'aqua'
 #satellite = 'snpp'
 
+version = '1.0dev0'
 delivery_id = '20170920-1'
 
 # Specify the intervals

utils/run_modis_airs_fusion_iff.py

@@ -24,7 +24,7 @@ import traceback
 import logging
 
 # every module should have a LOG object
-LOG = logging.getLogger(__file__)
+LOG = logging.getLogger(__name__)
 
 def setup_logging(verbosity):
     LOG.debug("Verbosity is {}".format(verbosity))

utils/run_viirs_cris_fusion_iff.py

@@ -24,7 +24,7 @@ import traceback
 import logging
 
 # every module should have a LOG object
-LOG = logging.getLogger(__file__)
+LOG = logging.getLogger(__name__)
 
 def setup_logging(verbosity):
     LOG.debug("Verbosity is {}".format(verbosity))