Changed all references to jpss1 to noaa20 for cluster submission scripts.

submit_fusion_matlab.py

@@ -28,8 +28,8 @@ setup_logging(2)
 #
 # General information
 #
-delivery_id = '20180620-1'
-version = '1.0dev3'
+delivery_id = '20181025-1'
+version = '1.0dev10'
 wedge = timedelta(seconds=1.)
 day = timedelta(days=1.)
 
@@ -38,18 +38,20 @@ day = timedelta(days=1.)
 #
 #satellite = 'aqua'
 #granule_length = timedelta(minutes=5)
-satellite = 'snpp'
+#satellite = 'snpp'
+satellite = 'noaa20'
 granule_length = timedelta(minutes=6)
 
 #
 # Specify the intervals
 #
 #granule = datetime(2015, 4, 17, 17, 55) # Aqua
-granule = datetime(2018, 2, 3, 0, 0) # SNPP
+#granule = datetime(2019, 3, 12, 0, 0) # SNPP
+granule = datetime(2019, 3, 11, 8, 0) # JPSS-1 / NOAA-20
 intervals = [
-    #TimeInterval(granule, granule + granule_length - wedge)
+    TimeInterval(granule, granule + granule_length - wedge)
     #TimeInterval(granule, granule + timedelta(hours=1) - wedge)
-    TimeInterval(granule, granule + timedelta(days=1) - wedge)
+    #TimeInterval(granule, granule + timedelta(days=1) - wedge)
     #TimeInterval(granule - timedelta(hours=1), granule + timedelta(hours=1))
     #TimeInterval(datetime(2014, 7, 1, 0, 5), datetime(2014, 8, 1) - wedge)
     #TimeInterval(datetime(2014, 7, 1, 0, 0), datetime(2014, 7, 1, 0, 10) - wedge)
@@ -58,7 +60,10 @@ intervals = [
 ]
 #intervals = []
 #years = 2018
-#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
+#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(7,9) ]
+#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in range(1,366)]
+#jdays = [1, 11, 12, 27, 29, 34, 44, 47, 48, 52, 57, 61, 68, 86, 115, 145, 174, 184, 186, 187, 193, 201, 213, 232, 243, 255, 270, 305, 323, 352, 357]
+#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in jdays]
 
 #
 # Initialize the computation
@@ -77,6 +82,12 @@ log_name = 'fusion_matlab_{}_s{}_e{}_c{}.log'.format(
     intervals[-1].right.strftime('%Y%m%d%H%M'),
     dt.strftime('%Y%m%d%H%M%S'))
 
+job_mods = {
+    'requests': [
+        'Memory=6000'
+    ]
+}
+
 try:
     for interval in intervals:
         LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
@@ -98,7 +109,7 @@ try:
 
             try:
                 job_nums = []
-                job_nums = safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts)
+                job_nums = safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts, job_mods=job_mods)
 
                 if job_nums != []:
                     #job_nums = range(len(contexts))
@@ -113,7 +124,10 @@ try:
             except Exception:
                 LOG.warning(traceback.format_exc())
 
-            #sleep(30.)
+            #sleep(600.) # Sleep 10 mins
+            #sleep(1600.) # Sleep half hour
+            #sleep(3600.) # Sleep one hour
+            #sleep(5.)
 
         LOG.info("Closing log file {}".format(log_name))
         file_obj.close()

submit_fusion_matlab_ql.py

@@ -22,44 +22,38 @@ setup_logging(2)
 #
 # General information
 #
-delivery_id = '20180620-1'
-version = '1.0dev2'
+delivery_id = '20181025-1'
+version = '1.0dev7'
 wedge = timedelta(seconds=1.)
 day = timedelta(days=1.)
 
-#job_mods = {}
-job_mods = {
-    'requirements': ['TARGET.Scratch >= RequestScratch'],
-    'requests': [
-        'Scratch=3',
-        'Memory=8000',
-    ],
-    'classads': ['HookKeyword=SCRATCH'],
-}
-
 #
 # Satellite specific information
 #
-satellite = 'snpp'
+#satellite = 'snpp'
+satellite = 'noaa20'
 granule_length = timedelta(minutes=6)
 
 #
 # Specify the intervals
 #
-granule = datetime(2018, 2, 3, 0, 0) # SNPP
-intervals = [
+#granule = datetime(2018, 7, 2, 0, 0) # SNPP
+#intervals = [
     #TimeInterval(granule, granule + granule_length - wedge)
     #TimeInterval(granule, granule + timedelta(hours=1) - wedge)
-    TimeInterval(granule, granule + timedelta(days=1) - wedge)
+    #TimeInterval(granule, granule + timedelta(days=1) - wedge)
     #TimeInterval(granule - timedelta(hours=1), granule + timedelta(hours=1))
     #TimeInterval(datetime(2014, 7, 1, 0, 5), datetime(2014, 8, 1) - wedge)
     #TimeInterval(datetime(2014, 7, 1, 0, 0), datetime(2014, 7, 1, 0, 10) - wedge)
     #TimeInterval(datetime(2015, 4, 1, 0, 0), datetime(2015, 5, 1, 0, 0) - wedge)
     #TimeInterval(datetime(2015, 4, 25, 13,  0), datetime(2015, 4, 25,  14,  0) - wedge)
-]
+#]
 #intervals = []
-#years = 2018
-#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
+years = 2019
+#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
+intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in range(1,5)]
+#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, 276),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, 302),'%Y%j')+day-wedge)]
+
 
 #
 # Initialize the computation
@@ -79,6 +73,15 @@ log_name = '/home/flo/geoffc/fusion_matlab_logs/fusion_matlab_ql_{}_s{}_e{}_c{}.
     intervals[-1].right.strftime('%Y%m%d%H%M'),
     dt.strftime('%Y%m%d%H%M%S'))
 
+job_mods = {
+    'requirements': ['TARGET.Scratch >= RequestScratch'],
+    'requests': [
+        'Scratch=3',
+        'Memory=8000',
+    ],
+    'classads': ['HookKeyword=SCRATCH'],
+}
+
 try:
     for interval in intervals:
         LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
@@ -92,8 +95,8 @@ try:
         contexts.sort()
 
         if contexts != []:
-            #for context in contexts:
-                #LOG.info(context)
+            for context in contexts:
+                LOG.info(context)
 
             LOG.info("\tFirst context: {}".format(contexts[0]))
             LOG.info("\tLast context:  {}".format(contexts[-1]))
@@ -109,13 +112,18 @@ try:
                     file_obj.write("contexts: [{}, {}]; job numbers: [{}..{}]\n".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
                     LOG.info("contexts: [{}, {}]; job numbers: [{},{}]".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
                     LOG.info("job numbers: [{}..{}]\n".format(job_nums[0],job_nums[-1]))
+                    #sleep(10800.) # Sleep three hours
+                    #sleep(7200.) # Sleep two hours
+                    sleep(3600.) # Sleep one hour
+                    #sleep(1800.) # Sleep half hour
+                    #sleep(600.) # Sleep ten minutes
                 else:
                     LOG.info("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
                     file_obj.write("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
+                    sleep(10.) # Sleep 10 seconds
             except Exception:
                 LOG.warning(traceback.format_exc())
 
-            #sleep(30.)
 
         LOG.info("Closing log file {}".format(log_name))
         file_obj.close()