Skip to content
Snippets Groups Projects
Commit 1a8c7e62 authored by Geoff Cureton's avatar Geoff Cureton
Browse files

Changed all references to jpss1 to noaa20 for cluster submission scripts.

parent c41eb541
No related branches found
No related tags found
No related merge requests found
......@@ -28,8 +28,8 @@ setup_logging(2)
#
# General information
#
delivery_id = '20180620-1'
version = '1.0dev3'
delivery_id = '20181025-1'
version = '1.0dev10'
wedge = timedelta(seconds=1.)
day = timedelta(days=1.)
......@@ -38,18 +38,20 @@ day = timedelta(days=1.)
#
#satellite = 'aqua'
#granule_length = timedelta(minutes=5)
satellite = 'snpp'
#satellite = 'snpp'
satellite = 'noaa20'
granule_length = timedelta(minutes=6)
#
# Specify the intervals
#
#granule = datetime(2015, 4, 17, 17, 55) # Aqua
granule = datetime(2018, 2, 3, 0, 0) # SNPP
#granule = datetime(2019, 3, 12, 0, 0) # SNPP
granule = datetime(2019, 3, 11, 8, 0) # JPSS-1 / NOAA-20
intervals = [
#TimeInterval(granule, granule + granule_length - wedge)
TimeInterval(granule, granule + granule_length - wedge)
#TimeInterval(granule, granule + timedelta(hours=1) - wedge)
TimeInterval(granule, granule + timedelta(days=1) - wedge)
#TimeInterval(granule, granule + timedelta(days=1) - wedge)
#TimeInterval(granule - timedelta(hours=1), granule + timedelta(hours=1))
#TimeInterval(datetime(2014, 7, 1, 0, 5), datetime(2014, 8, 1) - wedge)
#TimeInterval(datetime(2014, 7, 1, 0, 0), datetime(2014, 7, 1, 0, 10) - wedge)
......@@ -58,7 +60,10 @@ intervals = [
]
#intervals = []
#years = 2018
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(7,9) ]
#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in range(1,366)]
#jdays = [1, 11, 12, 27, 29, 34, 44, 47, 48, 52, 57, 61, 68, 86, 115, 145, 174, 184, 186, 187, 193, 201, 213, 232, 243, 255, 270, 305, 323, 352, 357]
#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in jdays]
#
# Initialize the computation
......@@ -77,6 +82,12 @@ log_name = 'fusion_matlab_{}_s{}_e{}_c{}.log'.format(
intervals[-1].right.strftime('%Y%m%d%H%M'),
dt.strftime('%Y%m%d%H%M%S'))
job_mods = {
'requests': [
'Memory=6000'
]
}
try:
for interval in intervals:
LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
......@@ -98,7 +109,7 @@ try:
try:
job_nums = []
job_nums = safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts)
job_nums = safe_submit_order(comp, [comp.dataset('fused_l1b')], contexts, job_mods=job_mods)
if job_nums != []:
#job_nums = range(len(contexts))
......@@ -113,7 +124,10 @@ try:
except Exception:
LOG.warning(traceback.format_exc())
#sleep(30.)
#sleep(600.) # Sleep 10 mins
#sleep(1600.) # Sleep half hour
#sleep(3600.) # Sleep one hour
#sleep(5.)
LOG.info("Closing log file {}".format(log_name))
file_obj.close()
......
......@@ -22,44 +22,38 @@ setup_logging(2)
#
# General information
#
delivery_id = '20180620-1'
version = '1.0dev2'
delivery_id = '20181025-1'
version = '1.0dev7'
wedge = timedelta(seconds=1.)
day = timedelta(days=1.)
#job_mods = {}
job_mods = {
'requirements': ['TARGET.Scratch >= RequestScratch'],
'requests': [
'Scratch=3',
'Memory=8000',
],
'classads': ['HookKeyword=SCRATCH'],
}
#
# Satellite specific information
#
satellite = 'snpp'
#satellite = 'snpp'
satellite = 'noaa20'
granule_length = timedelta(minutes=6)
#
# Specify the intervals
#
granule = datetime(2018, 2, 3, 0, 0) # SNPP
intervals = [
#granule = datetime(2018, 7, 2, 0, 0) # SNPP
#intervals = [
#TimeInterval(granule, granule + granule_length - wedge)
#TimeInterval(granule, granule + timedelta(hours=1) - wedge)
TimeInterval(granule, granule + timedelta(days=1) - wedge)
#TimeInterval(granule, granule + timedelta(days=1) - wedge)
#TimeInterval(granule - timedelta(hours=1), granule + timedelta(hours=1))
#TimeInterval(datetime(2014, 7, 1, 0, 5), datetime(2014, 8, 1) - wedge)
#TimeInterval(datetime(2014, 7, 1, 0, 0), datetime(2014, 7, 1, 0, 10) - wedge)
#TimeInterval(datetime(2015, 4, 1, 0, 0), datetime(2015, 5, 1, 0, 0) - wedge)
#TimeInterval(datetime(2015, 4, 25, 13, 0), datetime(2015, 4, 25, 14, 0) - wedge)
]
#]
#intervals = []
#years = 2018
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,calendar.monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
years = 2019
#intervals += [TimeInterval(datetime(years,month,1), datetime(years,month,monthrange(years,month)[1])+day-wedge) for month in range(1,3) ]
intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, jday),'%Y%j')+day-wedge) for jday in range(1,5)]
#intervals = [TimeInterval(datetime.strptime('{}{:03d}'.format(years, 276),'%Y%j'), datetime.strptime('{}{:03d}'.format(years, 302),'%Y%j')+day-wedge)]
#
# Initialize the computation
......@@ -79,6 +73,15 @@ log_name = '/home/flo/geoffc/fusion_matlab_logs/fusion_matlab_ql_{}_s{}_e{}_c{}.
intervals[-1].right.strftime('%Y%m%d%H%M'),
dt.strftime('%Y%m%d%H%M%S'))
job_mods = {
'requirements': ['TARGET.Scratch >= RequestScratch'],
'requests': [
'Scratch=3',
'Memory=8000',
],
'classads': ['HookKeyword=SCRATCH'],
}
try:
for interval in intervals:
LOG.info("Submitting interval {} -> {}".format(interval.left, interval.right))
......@@ -92,8 +95,8 @@ try:
contexts.sort()
if contexts != []:
#for context in contexts:
#LOG.info(context)
for context in contexts:
LOG.info(context)
LOG.info("\tFirst context: {}".format(contexts[0]))
LOG.info("\tLast context: {}".format(contexts[-1]))
......@@ -109,13 +112,18 @@ try:
file_obj.write("contexts: [{}, {}]; job numbers: [{}..{}]\n".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
LOG.info("contexts: [{}, {}]; job numbers: [{},{}]".format(contexts[0], contexts[-1], job_nums[0],job_nums[-1]))
LOG.info("job numbers: [{}..{}]\n".format(job_nums[0],job_nums[-1]))
#sleep(10800.) # Sleep three hours
#sleep(7200.) # Sleep two hours
sleep(3600.) # Sleep one hour
#sleep(1800.) # Sleep half hour
#sleep(600.) # Sleep ten minutes
else:
LOG.info("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
file_obj.write("contexts: [{}, {}]; --> no jobs\n".format(contexts[0], contexts[-1]))
sleep(10.) # Sleep 10 seconds
except Exception:
LOG.warning(traceback.format_exc())
#sleep(30.)
LOG.info("Closing log file {}".format(log_name))
file_obj.close()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment