Initial commit of the fusion_matlab package.

deploy/deploy.py

+import os
+from flo_deploy.packagelib import *
+
+
+class fusion_matlab(Package):
+    def deploy_package(self):
+        pass

example_local_prepare.py

+import sys
+from datetime import datetime, timedelta
+import traceback
+import logging
+
+from flo.time import TimeInterval
+from flo.ui import local_prepare, local_execute
+
+from flo.sw.fusion_matlab import FUSION_MATLAB
+from flo.sw.fusion_matlab.utils import setup_logging
+
+# every module should have a LOG object
+LOG = logging.getLogger(__file__)
+
+comp = FUSION_MATLAB()
+
+#
+# Local execution
+#
+
+# General information
+granule = datetime(2015, 4, 17, 14)
+interval = TimeInterval(granule, granule+timedelta(minutes=0))
+
+#satellite = 'snpp'
+satellite = 'aqua'
+delivery_id = '20170831-1'
+
+
+def local_execute_example(interval, satellite, delivery_id, skip_prepare=False, skip_execute=False, verbosity=2):
+
+    setup_logging(verbosity)
+
+    if satellite == 'snpp':
+        LOG.info("We are doing NPP...")
+    elif satellite == 'aqua':
+        LOG.info("We are doing AQUA...")
+    else:
+        LOG.error("Invalid satellite.")
+
+    # Get the required context...
+    contexts =  comp.find_contexts(interval, satellite, delivery_id)
+
+    if len(contexts) != 0:
+        LOG.info("Candidate contexts in interval...")
+        for context in contexts:
+            print("\t{}".format(context))
+
+        try:
+            if not skip_prepare:
+                LOG.info("Running fusion_matlab local_prepare()...")
+                local_prepare(comp, contexts[0]) 
+            if not skip_execute:
+                LOG.info("Running local_execute()...")
+                local_execute(comp, contexts[0])
+        except Exception, err:
+            LOG.error("{}".format(err))
+            LOG.debug(traceback.format_exc())
+    else:
+        LOG.error("There are no valid {} contexts for the interval {}.".format(satellite, interval))
+
+def print_contexts(interval, satellite, delivery_id):
+    contexts = comp.find_contexts(interval, satellite, delivery_id)
+    for context in contexts:
+        print context

source/flo/utils.py

+#!/usr/bin/env python
+# encoding: utf-8
+"""
+utils.py
+
+ * DESCRIPTION: This is the collection of utilities for parsing text, running external processes and
+ binaries, checking environments and whatever other mundane tasks aren't specific to this project.
+
+Created by Geoff Cureton <geoff.cureton@ssec.wisc.edu> on 2016-04-11.
+Copyright (c) 2011 University of Wisconsin Regents.
+Licensed under GNU GPLv3.
+"""
+
+import os
+import sys
+import string
+from copy import copy
+import shutil
+import traceback
+import logging
+
+
+LOG = logging.getLogger(__name__)
+
+def setup_logging(verbosity):
+    LOG.debug("Verbosity is {}".format(verbosity))
+
+    levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
+    level = levels[verbosity if verbosity < 4 else 3]
+
+    # Set up the logging
+    #console_logFormat = '%(asctime)s : %(name)-12s: %(levelname)-8s %(message)s'
+    #console_logFormat = '%(levelname)s:%(name)s:%(msg)s') # [%(filename)s:%(lineno)d]'
+    #console_logFormat = '%(asctime)s : %(funcName)s:%(lineno)d:  %(message)s'
+    #console_logFormat = '%(message)s'
+    console_logFormat = '(%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d:  %(message)s'
+    #console_logFormat = '%(asctime)s : (%(levelname)s):%(filename)s:%(funcName)s:%(lineno)d:' \
+        #' %(message)s'
+
+    dateFormat = '%Y-%m-%d %H:%M:%S'
+    logging.basicConfig(
+        stream=sys.stderr,
+        level=level,
+        format=console_logFormat,
+        datefmt=dateFormat)
+
+def cleanup(work_dir, objs_to_remove):
+    """
+    cleanup work directiory
+    remove evething except the products
+    """
+    for file_obj in objs_to_remove:
+        try:
+            if os.path.isdir(file_obj):
+                LOG.debug('Removing directory: {}'.format(file_obj))
+                shutil.rmtree(file_obj)
+            elif os.path.isfile(file_obj):
+                LOG.debug('Removing file: {}'.format(file_obj))
+                os.unlink(file_obj)
+        except Exception:
+            LOG.warn(traceback.format_exc())
+
+def link_files(dest_path, files):
+    '''
+    Link ancillary files into a destination directory.
+    '''
+    files_linked = 0
+    for src_file in files:
+        src = os.path.basename(src_file)
+        dest_file = os.path.join(dest_path, src)
+        if not os.path.exists(dest_file):
+            LOG.debug("Link {0} -> {1}".format(src_file, dest_file))
+            os.symlink(src_file, dest_file)
+            files_linked += 1
+        else:
+            LOG.warn('link already exists: {}'.format(dest_file))
+            files_linked += 1
+    return files_linked
+
+class SipsEnvironment(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+def make_time_stamp_d(timeObj):
+    '''
+    Returns a timestamp ending in deciseconds
+    '''
+    dateStamp = timeObj.strftime("%Y-%m-%d")
+    # seconds = repr(int(round(timeObj.second + float(timeObj.microsecond) / 1000000.)))
+    deciSeconds = int(round(float(timeObj.microsecond) / 100000.))
+    deciSeconds = repr(0 if deciSeconds > 9 else deciSeconds)
+    timeStamp = "{}.{}".format(timeObj.strftime("%H:%M:%S"), deciSeconds)
+    return "{} {}".format(dateStamp, timeStamp)
+
+def make_time_stamp_m(timeObj):
+    """
+    Returns a timestamp ending in milliseconds
+    """
+    dateStamp = timeObj.strftime("%Y-%m-%d")
+    # seconds = repr(int(round(timeObj.second + float(timeObj.microsecond) / 1000000.)))
+    milliseconds = int(round(float(timeObj.microsecond) / 1000.))
+    milliseconds = repr(000 if milliseconds > 999 else milliseconds)
+    timeStamp = "{}.{}".format(timeObj.strftime("%H:%M:%S"), str(milliseconds).zfill(3))
+    return "{} {}".format(dateStamp, timeStamp)
+
+def execution_time(startTime, endTime):
+    '''
+    Converts a time duration in seconds to days, hours, minutes etc...
+    '''
+
+    time_dict = {}
+
+    delta = endTime - startTime
+    days, remainder = divmod(delta, 86400.)
+    hours, remainder = divmod(remainder, 3600.)
+    minutes, seconds = divmod(remainder, 60.)
+
+    time_dict['delta'] = delta
+    time_dict['days'] = int(days)
+    time_dict['hours'] = int(hours)
+    time_dict['minutes'] = int(minutes)
+    time_dict['seconds'] = seconds
+
+    return time_dict
+
+def create_dir(dir):
+    '''
+    Create a directory
+    '''
+    returned_dir = copy(dir)
+    LOG.debug("We want to create the dir {} ...".format(dir))
+
+    try:
+        if returned_dir is not None:
+            returned_dir_path = os.path.dirname(returned_dir)
+            returned_dir_base = os.path.basename(returned_dir)
+            LOG.debug("returned_dir_path = {}".format(returned_dir_path))
+            LOG.debug("returned_dir_base = {}".format(returned_dir_base))
+            # Check if a directory and has write permissions...
+            if not os.path.exists(returned_dir) and os.access(returned_dir_path, os.W_OK):
+                LOG.debug("Creating directory {} ...".format(returned_dir))
+                os.makedirs(returned_dir)
+                # Check if the created dir has write permissions
+                if not os.access(returned_dir, os.W_OK):
+                    msg = "Created dir {} is not writable.".format(returned_dir)
+                    raise SipsEnvironment(msg)
+            elif os.path.exists(returned_dir):
+                LOG.debug("Directory {} exists...".format(returned_dir))
+                if not(os.path.isdir(returned_dir) and os.access(returned_dir, os.W_OK)):
+                    msg = "Existing dir {} is not writable.".format(returned_dir)
+                    raise SipsEnvironment(msg)
+            else:
+                raise SipsEnvironment("Cannot create {}".format(returned_dir))
+    except SipsEnvironment:
+        LOG.debug("Unable to create {}".format(returned_dir))
+        LOG.debug(traceback.format_exc())
+        returned_dir = None
+    except OSError:
+        LOG.debug(
+            "Unable to create new dir '{}' in {}".format(returned_dir_base, returned_dir_path))
+        LOG.debug(traceback.format_exc())
+        returned_dir = None
+    except Exception:
+        LOG.warning("General error for {}".format(returned_dir))
+        LOG.debug(traceback.format_exc())
+        returned_dir = None
+
+    LOG.debug('Final returned_dir = {}'.format(returned_dir))
+    return returned_dir

submit_fusion_matlab.py

+import sys
+from datetime import datetime, timedelta
+import traceback
+import logging
+
+from flo.time import TimeInterval
+from flo.ui import local_prepare, local_execute
+
+from flo.sw.fusion_matlab import FUSION_MATLAB
+from flo.sw.fusion_matlab.utils import setup_logging
+
+# every module should have a LOG object
+LOG = logging.getLogger(__file__)
+
+setup_logging(2)
+
+comp = FUSION_MATLAB()
+
+satellite = 'snpp'
+#satellite = 'aqua'
+delivery_id = '20170831-1'
+
+# Specify the intervals
+wedge = timedelta(seconds=1.)
+intervals = [
+    # TimeInterval(datetime(2014, 1, 1, 0, 0), datetime(2014, 2, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 2, 1, 0, 0), datetime(2014, 3, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 3, 1, 0, 0), datetime(2014, 4, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 4, 1, 0, 0), datetime(2014, 5, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 5, 1, 0, 0), datetime(2014, 6, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 6, 1, 0, 0), datetime(2014, 7, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 7, 1, 0, 0), datetime(2014, 8, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 8, 1, 0, 0), datetime(2014, 9, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 9, 1, 0, 0), datetime(2014, 10, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 10, 1, 0, 0), datetime(2014, 11, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 11, 1, 0, 0), datetime(2014, 12, 1, 0, 0) - wedge),
+    #TimeInterval(datetime(2014, 12, 1, 0, 0), datetime(2015, 1, 1, 0, 0) - wedge)
+    # TimeInterval(datetime(2014, 1, 1, 0, 0), datetime(2014, 2, 1, 0, 0) - wedge)
+    #TimeInterval(datetime(2016, 5, 17, 5, 30), datetime(2016, 5, 17, 6, 0) - wedge),
+    TimeInterval(datetime(2016, 5, 18, 3, 36), datetime(2016, 5, 18, 3, 49) - wedge)
+]
+
+LOG.info("Submitting intervals...")
+for interval in intervals:
+    LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
+    
+    contexts =  comp.find_contexts(interval, satellite, delivery_id)
+
+    LOG.info("\tThere are {} contexts in this interval".format(len(contexts)))
+    contexts.sort()
+    
+    for context in contexts:
+        print context
+
+    LOG.info("\tFirst context: {}".format(contexts[0]))
+    LOG.info("\tLast context:  {}".format(contexts[-1]))
+    #LOG.info("\t{}".format(safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts))
+    #time.sleep(30.)