Updating the fusion_matlab package.

example_local_prepare.py

@@ -23,8 +23,8 @@ comp = FUSION_MATLAB()
 
 satellite = 'snpp'
 #satellite = 'aqua'
-delivery_id = '20170920-1'
-version = '1.0dev1'
+delivery_id = '20180225-1'
+version = '1.0dev2'
 
 
 def local_execute_example(interval, satellite, version, skip_prepare=False, skip_execute=False, verbosity=2):

submit_fusion_matlab.py

+#!/usr/bin/env python
+# encoding: utf-8
+
 import sys
 import traceback
 import calendar
 import logging
+from calendar import monthrange
+from time import sleep
 
 from flo.ui import safe_submit_order
 from timeutil import TimeInterval, datetime, timedelta
@@ -14,22 +19,27 @@ LOG = logging.getLogger(__name__)
 
 setup_logging(2)
 
-comp = FUSION_MATLAB()
+#
+# General information
+#
+delivery_id = '20180225-1'
+version = '1.0dev2'
+wedge = timedelta(seconds=1.)
+day = timedelta(days=1.)
 
+#
+# Satellite specific information
+#
 #satellite = 'aqua'
 #granule_length = timedelta(minutes=5)
 satellite = 'snpp'
 granule_length = timedelta(minutes=6)
 
-version = '1.0dev0'
-delivery_id = '20170920-1'
-version = '1.0dev0'
-
+#
 # Specify the intervals
-#granule = datetime(2014, 7, 6, 2, 0) # Gala Wind
-granule = datetime(2015, 4, 17) # Bryan Baum
-#granule = datetime(2015, 4, 17, 17, 55) # Bryan Baum
-wedge = timedelta(seconds=1.)
+#
+#granule = datetime(2015, 4, 17, 17, 55) # Aqua
+granule = datetime(2018, 2, 2, 18, 36) # SNPP
 #intervals = [
     #TimeInterval(granule, granule + granule_length - wedge)
     #TimeInterval(granule, granule + timedelta(hours=1) - wedge)
@@ -40,23 +50,67 @@ wedge = timedelta(seconds=1.)
     #TimeInterval(datetime(2015, 4, 1, 0, 0), datetime(2015, 5, 1, 0, 0) - wedge)
     #TimeInterval(datetime(2015, 4, 25, 13,  0), datetime(2015, 4, 25,  14,  0) - wedge)
 #]
+intervals = []
+years = 2018
+intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
 
-intervals = [TimeInterval(datetime(2015, 4, day), datetime(2015, 4, day, 23, 59)) for day in range(1, calendar.monthrange(2015, 4)[1]+1)]
-
+#
+# Initialize the computation
+#
+comp = FUSION_MATLAB()
 
+#
+# Submit the jobs
+#
 LOG.info("Submitting intervals...")
-for interval in intervals:
-    LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
 
-    contexts =  comp.find_contexts(interval, satellite, version)
+dt = datetime.utcnow()
+log_name = '/home/flo/geoffc/fusion_matlab_logs/fusion_matlab_{}_s{}_e{}_c{}.log'.format(
+    satellite,
+    intervals[0].left.strftime('%Y%m%d%H%M'),
+    intervals[-1].right.strftime('%Y%m%d%H%M'),
+    dt.strftime('%Y%m%d%H%M%S'))
+
+try:
+    for interval in intervals:
+        LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
+
+        contexts =  comp.find_contexts(interval, satellite, version)
+
+        LOG.info("Opening log file {}".format(log_name))
+        file_obj = open(log_name,'a')
+
+        LOG.info("\tThere are {} contexts in this interval".format(len(contexts)))
+        contexts.sort()
+
+        if contexts != []:
+            #for context in contexts:
+                #LOG.info(context)
+
+            LOG.info("\tFirst context: {}".format(contexts[0]))
+            LOG.info("\tLast context:  {}".format(contexts[-1]))
+
+            try:
+                job_nums = []
+                job_nums = safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts)
+
+                if job_nums != []:
+                    #job_nums = range(len(contexts))
+                    #LOG.info("\t{}".format(job_nums))
+
+                    file_obj.write("contexts: [{}, {}]; job numbers: [{}..{}]\n".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
+                    LOG.info("contexts: [{}, {}]; job numbers: [{},{}]".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
+                    LOG.info("job numbers: [{}..{}]\n".format(job_nums[0],job_nums[-1]))
+                else:
+                    LOG.info("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
+                    file_obj.write("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
+            except Exception:
+                LOG.warning(traceback.format_exc())
 
-    LOG.info("\tThere are {} contexts in this interval".format(len(contexts)))
-    contexts.sort()
+            #sleep(30.)
 
-    #for context in contexts:
-        #print context
+        LOG.info("Closing log file {}".format(log_name))
+        file_obj.close()
 
-    LOG.info("\tFirst context: {}".format(contexts[0]))
-    LOG.info("\tLast context:  {}".format(contexts[-1]))
-    LOG.info("\t{}".format(safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts)))
-    #time.sleep(30.)
+except Exception:
+    LOG.warning(traceback.format_exc())